From 48d3e8d3503e8c64048f158e1c2c5206cdefd9c6 Mon Sep 17 00:00:00 2001 From: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com> Date: Wed, 24 Jun 2020 18:28:53 -0700 Subject: [PATCH 01/38] features: Add install failure tracking removal through `spack clean` (#15314) * Add ability to force removal of install failure tracking data through spack clean * Add clean failures option to packaging guide --- lib/spack/docs/packaging_guide.rst | 22 +++++++----- lib/spack/spack/cmd/clean.py | 17 ++++++--- lib/spack/spack/database.py | 26 +++++++++++--- lib/spack/spack/installer.py | 9 ++++- lib/spack/spack/test/cmd/clean.py | 16 +++++---- lib/spack/spack/test/cmd/install.py | 20 +++++------ lib/spack/spack/test/conftest.py | 10 ++++++ lib/spack/spack/test/installer.py | 54 ++++++++++++++++++++++++++++- share/spack/spack-completion.bash | 2 +- 9 files changed, 140 insertions(+), 36 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 1f246c0faad..533036852b3 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -4252,23 +4252,29 @@ Does this in one of two ways: ``spack clean`` ^^^^^^^^^^^^^^^ -Cleans up all of Spack's temporary and cached files. This can be used to +Cleans up Spack's temporary and cached files. This command can be used to recover disk space if temporary files from interrupted or failed installs -accumulate in the staging area. +accumulate. When called with ``--stage`` or without arguments this removes all staged files. -When called with ``--downloads`` this will clear all resources -:ref:`cached ` during installs. +The ``--downloads`` option removes cached :ref:`cached ` downloads. -When called with ``--user-cache`` this will remove caches in the user home -directory, including cached virtual indices. +You can force the removal of all install failure tracking markers using the +``--failures`` option. Note that ``spack install`` will automatically clear +relevant failure markings prior to performing the requested installation(s). + +Long-lived caches, like the virtual package index, are removed using the +``--misc-cache`` option. + +The ``--python-cache`` option removes `.pyc`, `.pyo`, and `__pycache__` +folders. To remove all of the above, the command can be called with ``--all``. -When called with positional arguments, cleans up temporary files only -for a particular package. If ``fetch``, ``stage``, or ``install`` +When called with positional arguments, this command cleans up temporary files +only for a particular package. If ``fetch``, ``stage``, or ``install`` are run again after this, Spack's build process will start from scratch. diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 791a1b7dc31..d847e7a7c07 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -23,9 +23,9 @@ class AllClean(argparse.Action): - """Activates flags -s -d -m and -p simultaneously""" + """Activates flags -s -d -f -m and -p simultaneously""" def __call__(self, parser, namespace, values, option_string=None): - parser.parse_args(['-sdmp'], namespace=namespace) + parser.parse_args(['-sdfmp'], namespace=namespace) def setup_parser(subparser): @@ -35,6 +35,9 @@ def setup_parser(subparser): subparser.add_argument( '-d', '--downloads', action='store_true', help="remove cached downloads") + subparser.add_argument( + '-f', '--failures', action='store_true', + help="force removal of all install failure tracking markers") subparser.add_argument( '-m', '--misc-cache', action='store_true', help="remove long-lived caches, like the virtual package index") @@ -42,15 +45,15 @@ def setup_parser(subparser): '-p', '--python-cache', action='store_true', help="remove .pyc, .pyo files and __pycache__ folders") subparser.add_argument( - '-a', '--all', action=AllClean, help="equivalent to -sdmp", nargs=0 + '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0 ) arguments.add_common_arguments(subparser, ['specs']) def clean(parser, args): # If nothing was set, activate the default - if not any([args.specs, args.stage, args.downloads, args.misc_cache, - args.python_cache]): + if not any([args.specs, args.stage, args.downloads, args.failures, + args.misc_cache, args.python_cache]): args.stage = True # Then do the cleaning falling through the cases @@ -70,6 +73,10 @@ def clean(parser, args): tty.msg('Removing cached downloads') spack.caches.fetch_cache.destroy() + if args.failures: + tty.msg('Removing install failure marks') + spack.installer.clear_failures() + if args.misc_cache: tty.msg('Removing cached information on repositories') spack.caches.misc_cache.destroy() diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 001f96d1ff3..2acc8c2db62 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -23,6 +23,7 @@ import contextlib import datetime import os +import six import socket import sys import time @@ -33,14 +34,14 @@ _use_uuid = False pass +import llnl.util.filesystem as fs import llnl.util.tty as tty -import six + import spack.repo import spack.spec import spack.store import spack.util.lock as lk import spack.util.spack_json as sjson -from llnl.util.filesystem import mkdirp from spack.directory_layout import DirectoryLayoutError from spack.error import SpackError from spack.filesystem_view import YamlFilesystemView @@ -316,10 +317,10 @@ def __init__(self, root, db_dir=None, upstream_dbs=None, # Create needed directories and files if not os.path.exists(self._db_dir): - mkdirp(self._db_dir) + fs.mkdirp(self._db_dir) if not os.path.exists(self._failure_dir) and not is_upstream: - mkdirp(self._failure_dir) + fs.mkdirp(self._failure_dir) self.is_upstream = is_upstream self.last_seen_verifier = '' @@ -373,6 +374,23 @@ def _failed_spec_path(self, spec): return os.path.join(self._failure_dir, '{0}-{1}'.format(spec.name, spec.full_hash())) + def clear_all_failures(self): + """Force remove install failure tracking files.""" + tty.debug('Releasing prefix failure locks') + for pkg_id in list(self._prefix_failures.keys()): + lock = self._prefix_failures.pop(pkg_id, None) + if lock: + lock.release_write() + + # Remove all failure markings (aka files) + tty.debug('Removing prefix failure tracking files') + for fail_mark in os.listdir(self._failure_dir): + try: + os.remove(os.path.join(self._failure_dir, fail_mark)) + except OSError as exc: + tty.warn('Unable to remove failure marking file {0}: {1}' + .format(fail_mark, str(exc))) + def clear_failure(self, spec, force=False): """ Remove any persistent and cached failure tracking for the spec. diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index cd2aa00bd02..a2b0220a8ba 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -373,6 +373,13 @@ def _update_explicit_entry_in_db(pkg, rec, explicit): rec.explicit = True +def clear_failures(): + """ + Remove all failure tracking markers for the Spack instance. + """ + spack.store.db.clear_all_failures() + + def dump_packages(spec, path): """ Dump all package information for a spec and its dependencies. @@ -835,7 +842,7 @@ def _cleanup_failed(self, pkg_id): """ lock = self.failed.get(pkg_id, None) if lock is not None: - err = "{0} exception when removing failure mark for {1}: {2}" + err = "{0} exception when removing failure tracking for {1}: {2}" msg = 'Removing failure mark on {0}' try: tty.verbose(msg.format(pkg_id)) diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index c3e9d79f239..4df708cf509 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -28,18 +28,21 @@ def __call__(self, *args, **kwargs): spack.caches.fetch_cache, 'destroy', Counter(), raising=False) monkeypatch.setattr( spack.caches.misc_cache, 'destroy', Counter()) + monkeypatch.setattr( + spack.installer, 'clear_failures', Counter()) @pytest.mark.usefixtures( 'mock_packages', 'config', 'mock_calls_for_clean' ) @pytest.mark.parametrize('command_line,counters', [ - ('mpileaks', [1, 0, 0, 0]), - ('-s', [0, 1, 0, 0]), - ('-sd', [0, 1, 1, 0]), - ('-m', [0, 0, 0, 1]), - ('-a', [0, 1, 1, 1]), - ('', [0, 0, 0, 0]), + ('mpileaks', [1, 0, 0, 0, 0]), + ('-s', [0, 1, 0, 0, 0]), + ('-sd', [0, 1, 1, 0, 0]), + ('-m', [0, 0, 0, 1, 0]), + ('-f', [0, 0, 0, 0, 1]), + ('-a', [0, 1, 1, 1, 1]), + ('', [0, 0, 0, 0, 0]), ]) def test_function_calls(command_line, counters): @@ -52,3 +55,4 @@ def test_function_calls(command_line, counters): assert spack.stage.purge.call_count == counters[1] assert spack.caches.fetch_cache.destroy.call_count == counters[2] assert spack.caches.misc_cache.destroy.call_count == counters[3] + assert spack.installer.clear_failures.call_count == counters[4] diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 4a5db55c015..e4df22a6a56 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -610,17 +610,17 @@ def test_build_warning_output(tmpdir, mock_fetch, install_mockery, capfd): assert 'foo.c:89: warning: some weird warning!' in msg -def test_cache_only_fails(tmpdir, mock_fetch, install_mockery, capfd): - msg = '' - with capfd.disabled(): - try: - install('--cache-only', 'libdwarf') - except spack.installer.InstallError as e: - msg = str(e) +def test_cache_only_fails(tmpdir, mock_fetch, install_mockery): + # libelf from cache fails to install, which automatically removes the + # the libdwarf build task and flags the package as failed to install. + err_msg = 'Installation of libdwarf failed' + with pytest.raises(spack.installer.InstallError, match=err_msg): + install('--cache-only', 'libdwarf') - # libelf from cache failed to install, which automatically removed the - # the libdwarf build task and flagged the package as failed to install. - assert 'Installation of libdwarf failed' in msg + # Check that failure prefix locks are still cached + failure_lock_prefixes = ','.join(spack.store.db._prefix_failures.keys()) + assert 'libelf' in failure_lock_prefixes + assert 'libdwarf' in failure_lock_prefixes def test_install_only_dependencies(tmpdir, mock_fetch, install_mockery): diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index bac5bcebf60..8bd8deadce3 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -601,6 +601,16 @@ def install_mockery(tmpdir, config, mock_packages, monkeypatch): tmpdir.join('opt').remove() spack.store.store = real_store + # Also wipe out any cached prefix failure locks (associated with + # the session-scoped mock archive). + for pkg_id in list(spack.store.db._prefix_failures.keys()): + lock = spack.store.db._prefix_failures.pop(pkg_id, None) + if lock: + try: + lock.release_write() + except Exception: + pass + @pytest.fixture(scope='function') def install_mockery_mutable_config( diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py index 89efbe4fbec..cc4b168e6c8 100644 --- a/lib/spack/spack/test/installer.py +++ b/lib/spack/spack/test/installer.py @@ -462,6 +462,58 @@ def _repoerr(repo, name): assert "Couldn't copy in provenance for cmake" in out +def test_clear_failures_success(install_mockery): + """Test the clear_failures happy path.""" + + # Set up a test prefix failure lock + lock = lk.Lock(spack.store.db.prefix_fail_path, start=1, length=1, + default_timeout=1e-9, desc='test') + try: + lock.acquire_write() + except lk.LockTimeoutError: + tty.warn('Failed to write lock the test install failure') + spack.store.db._prefix_failures['test'] = lock + + # Set up a fake failure mark (or file) + fs.touch(os.path.join(spack.store.db._failure_dir, 'test')) + + # Now clear failure tracking + inst.clear_failures() + + # Ensure there are no cached failure locks or failure marks + assert len(spack.store.db._prefix_failures) == 0 + assert len(os.listdir(spack.store.db._failure_dir)) == 0 + + # Ensure the core directory and failure lock file still exist + assert os.path.isdir(spack.store.db._failure_dir) + assert os.path.isfile(spack.store.db.prefix_fail_path) + + +def test_clear_failures_errs(install_mockery, monkeypatch, capsys): + """Test the clear_failures exception paths.""" + orig_fn = os.remove + err_msg = 'Mock os remove' + + def _raise_except(path): + raise OSError(err_msg) + + # Set up a fake failure mark (or file) + fs.touch(os.path.join(spack.store.db._failure_dir, 'test')) + + monkeypatch.setattr(os, 'remove', _raise_except) + + # Clear failure tracking + inst.clear_failures() + + # Ensure expected warning generated + out = str(capsys.readouterr()[1]) + assert 'Unable to remove failure' in out + assert err_msg in out + + # Restore remove for teardown + monkeypatch.setattr(os, 'remove', orig_fn) + + def test_check_deps_status_install_failure(install_mockery, monkeypatch): spec, installer = create_installer('a') @@ -669,7 +721,7 @@ def _raise_except(lock): installer._cleanup_failed(pkg_id) out = str(capsys.readouterr()[1]) - assert 'exception when removing failure mark' in out + assert 'exception when removing failure tracking' in out assert msg in out diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 620dd9bef2f..e2db47ba301 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -484,7 +484,7 @@ _spack_ci_rebuild() { _spack_clean() { if $list_options then - SPACK_COMPREPLY="-h --help -s --stage -d --downloads -m --misc-cache -p --python-cache -a --all" + SPACK_COMPREPLY="-h --help -s --stage -d --downloads -f --failures -m --misc-cache -p --python-cache -a --all" else _all_packages fi From e12784bcb1fbc744e3407de17818c3113ab2d7fd Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 24 Jun 2020 19:57:37 -0700 Subject: [PATCH 02/38] lwgrp: add v1.0.3 (#17242) --- var/spack/repos/builtin/packages/lwgrp/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/lwgrp/package.py b/var/spack/repos/builtin/packages/lwgrp/package.py index 4356f211a8f..df13c4674ad 100644 --- a/var/spack/repos/builtin/packages/lwgrp/package.py +++ b/var/spack/repos/builtin/packages/lwgrp/package.py @@ -13,6 +13,7 @@ class Lwgrp(AutotoolsPackage): homepage = "https://github.com/hpc/lwgrp" url = "https://github.com/hpc/lwgrp/releases/download/v1.0.2/lwgrp-1.0.2.tar.gz" + version('1.0.3', sha256='20b2fc3908bfdf04d1c177f86e227a147214cd155c548b3dd75e54c78e1c1c47') version('1.0.2', sha256='c9d4233946e40f01efd0b4644fd9224becec51b9b5f8cbf45f5bac3129b5b536') depends_on('mpi') From b00bc2a18eba001bf82b5b67ef61ca97c39a1c43 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Wed, 24 Jun 2020 19:58:09 -0700 Subject: [PATCH 03/38] dtcmp: add v1.1.1 (#17240) Signed-off-by: Adam Moody --- var/spack/repos/builtin/packages/dtcmp/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/dtcmp/package.py b/var/spack/repos/builtin/packages/dtcmp/package.py index 1ff4c237758..e9e17c06f91 100644 --- a/var/spack/repos/builtin/packages/dtcmp/package.py +++ b/var/spack/repos/builtin/packages/dtcmp/package.py @@ -13,6 +13,7 @@ class Dtcmp(AutotoolsPackage): homepage = "https://github.com/hpc/dtcmp" url = "https://github.com/hpc/dtcmp/releases/download/v1.0.3/dtcmp-1.0.3.tar.gz" + version('1.1.1', sha256='ddf3c57cbb83515e1b7e4111b8a83f832e66376b40eee5d8a5549dd7b8446bc6') version('1.1.0', sha256='fd2c4485eee560a029f62c8f227df4acdb1edc9340907f4ae2dbee59f05f057d') version('1.0.3', sha256='1327368e2808043ad5f245cd16f0da19543de50eae02a4e22b8a1c2e0eff8f35') From ad708cde53cf19b807402f85458d930167406fba Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Jun 2020 21:58:31 -0500 Subject: [PATCH 04/38] GPyTorch: add new package (#17237) --- .../builtin/packages/py-gpytorch/package.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-gpytorch/package.py diff --git a/var/spack/repos/builtin/packages/py-gpytorch/package.py b/var/spack/repos/builtin/packages/py-gpytorch/package.py new file mode 100644 index 00000000000..abd728a8c5b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-gpytorch/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyGpytorch(PythonPackage): + """GPyTorch is a Gaussian process library implemented using PyTorch. + GPyTorch is designed for creating scalable, flexible, and modular Gaussian + process models with ease.""" + + homepage = "https://gpytorch.ai/" + url = "https://pypi.io/packages/source/g/gpytorch/gpytorch-1.1.1.tar.gz" + + maintainers = ['adamjstewart'] + + version('1.1.1', sha256='76bd455db2f17af5425f73acfaa6d61b8adb1f07ad4881c0fa22673f84fb571a') + + depends_on('python@3.6:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-torch@1.5:', type=('build', 'run')) From cd2e945e4e2626d71b3f6d6b54924b3e6de5cae7 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Jun 2020 21:59:10 -0500 Subject: [PATCH 05/38] py-horovod: add latest version (#17235) --- var/spack/repos/builtin/packages/py-horovod/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-horovod/package.py b/var/spack/repos/builtin/packages/py-horovod/package.py index 7d6a28939f2..a4cb5ed1b84 100644 --- a/var/spack/repos/builtin/packages/py-horovod/package.py +++ b/var/spack/repos/builtin/packages/py-horovod/package.py @@ -14,6 +14,7 @@ class PyHorovod(PythonPackage): maintainers = ['adamjstewart'] version('master', branch='master', submodules=True) + version('0.19.5', tag='v0.19.5', submodules=True) version('0.19.4', tag='v0.19.4', submodules=True) version('0.19.3', tag='v0.19.3', submodules=True) version('0.19.2', tag='v0.19.2', submodules=True) From 791267c3dcd539f604a86f381582db5e28439b9f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Jun 2020 21:59:32 -0500 Subject: [PATCH 06/38] NCCL: add latest version (#17234) --- var/spack/repos/builtin/packages/nccl/package.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/nccl/package.py b/var/spack/repos/builtin/packages/nccl/package.py index 74a45ba13ab..d1f4de962db 100644 --- a/var/spack/repos/builtin/packages/nccl/package.py +++ b/var/spack/repos/builtin/packages/nccl/package.py @@ -6,14 +6,15 @@ from spack import * -class Nccl(MakefilePackage): +class Nccl(MakefilePackage, CudaPackage): """Optimized primitives for collective multi-GPU communication.""" homepage = "https://github.com/NVIDIA/nccl" - url = "https://github.com/NVIDIA/nccl/archive/v2.6.4-1.tar.gz" + url = "https://github.com/NVIDIA/nccl/archive/v2.7.3-1.tar.gz" maintainers = ['adamjstewart'] + version('2.7.3-1', sha256='dc7b8794373306e323363314c3327796e416f745e8003490fc1407a22dd7acd6') version('2.6.4-1', sha256='ed8c9dfd40e013003923ae006787b1a30d3cb363b47d2e4307eaa2624ebba2ba') version('2.5.7-1', sha256='781a6bb2278566be4abbdf22b2fa19afc7306cff4b312c82bd782979b368014e') version('2.5.6-2', sha256='8a30e0b4813a825592872fcbeeede22a659e2c399074dcce02960591dc81387d') @@ -26,12 +27,15 @@ class Nccl(MakefilePackage): version('1.3.4-1', sha256='11e4eb44555bb28b9cbad973dacb4640b82710c9769e719afc2013b63ffaf884') version('1.3.0-1', sha256='53f36151061907bdcafad1c26c1d9370a0a8400f561a83704a5138213ba51003') - depends_on('cuda') + variant('cuda', default=True, description='Build with CUDA') + depends_on('rdma-core', when='@2.3.5-5:') # https://github.com/NVIDIA/nccl/issues/244 patch('so_reuseport.patch', when='@2.3.7-1:2.4.8-1') + conflicts('~cuda', msg='NCCL requires CUDA') + @property def build_targets(self): return ['CUDA_HOME={0}'.format(self.spec['cuda'].prefix)] From 95f351de7ef2baab162c16df3fe18299f66b19a8 Mon Sep 17 00:00:00 2001 From: Michio Ogawa Date: Thu, 25 Jun 2020 12:09:44 +0900 Subject: [PATCH 07/38] Updated MUMPS(5.3.3) (#17223) * Updated MUMPS-5.3.3 * fixed E741 error(l -> ltr) --- var/spack/repos/builtin/packages/mumps/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 5a5110d1e17..773f7e9b1fc 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -15,6 +15,7 @@ class Mumps(Package): homepage = "http://mumps.enseeiht.fr" url = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz" + version('5.3.3', sha256='27e7749ac05006bf8e81a457c865402bb72a42bf3bc673da49de1020f0f32011') version('5.2.0', sha256='41f2c7cb20d69599fb47e2ad6f628f3798c429f49e72e757e70722680f70853f') version('5.1.2', sha256='eb345cda145da9aea01b851d17e54e7eef08e16bfa148100ac1f7f046cd42ae9') version('5.1.1', sha256='a2a1f89c470f2b66e9982953cbd047d429a002fab9975400cef7190d01084a06') @@ -265,9 +266,9 @@ def install(self, spec, prefix): letters_variants = [ ['s', '+float'], ['c', '+complex+float'], ['d', '+double'], ['z', '+complex+double']] - for l, v in letters_variants: + for ltr, v in letters_variants: if v in spec: - make(l + 'examples') + make(ltr + 'examples') install_tree('lib', prefix.lib) install_tree('include', prefix.include) From bed11a5a5454ad4693164c647bfccaa7edfbe48a Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Wed, 24 Jun 2020 20:10:43 -0700 Subject: [PATCH 08/38] timemory: Updated nearly all options to default to OFF (#17221) * Updated nearly all options to default to OFF * Fixed imported but unused module flake error --- .../builtin/packages/timemory/package.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/var/spack/repos/builtin/packages/timemory/package.py b/var/spack/repos/builtin/packages/timemory/package.py index 8561ca07f31..d54ccf4fba7 100644 --- a/var/spack/repos/builtin/packages/timemory/package.py +++ b/var/spack/repos/builtin/packages/timemory/package.py @@ -6,7 +6,6 @@ # ---------------------------------------------------------------------------- from spack import * -from sys import platform class Timemory(CMakePackage): @@ -23,28 +22,26 @@ class Timemory(CMakePackage): version('3.0.0', commit='b36b1673b2c6b7ff3126d8261bef0f8f176c7beb', submodules=True) - linux = False if platform == 'darwin' else True - variant('shared', default=True, description='Build shared libraries') variant('static', default=False, description='Build static libraries') - variant('python', default=True, description='Enable Python support') - variant('mpi', default=True, description='Enable MPI support') + variant('python', default=False, description='Enable Python support') + variant('mpi', default=False, description='Enable MPI support') variant('tau', default=False, description='Enable TAU support') - variant('papi', default=linux, description='Enable PAPI support') - variant('cuda', default=linux, description='Enable CUDA support') - variant('cupti', default=linux, description='Enable CUPTI support') + variant('papi', default=False, description='Enable PAPI support') + variant('cuda', default=False, description='Enable CUDA support') + variant('cupti', default=False, description='Enable CUPTI support') variant('tools', default=True, description='Build/install extra tools') variant('vtune', default=False, description='Enable VTune support') variant('upcxx', default=False, description='Enable UPC++ support') - variant('gotcha', default=linux, description='Enable GOTCHA support') - variant('likwid', default=linux, description='Enable LIKWID support') + variant('gotcha', default=False, description='Enable GOTCHA support') + variant('likwid', default=False, description='Enable LIKWID support') variant('caliper', default=False, description='Enable Caliper support') - variant('dyninst', default=linux, + variant('dyninst', default=False, description='Build dynamic instrumentation tools') variant('examples', default=False, description='Build/install examples') - variant('gperftools', default=True, + variant('gperftools', default=False, description='Enable gperftools support') - variant('kokkos_tools', default=True, + variant('kokkos_tools', default=False, description=('Build generic kokkos-tools libraries, e.g. ' 'kp_timemory, kp_timemory_filter')) variant('kokkos_build_config', default=False, @@ -72,19 +69,19 @@ class Timemory(CMakePackage): description='Build timemory with extra optimization flags') variant('cxxstd', default='14', description='C++ language standard', values=('14', '17', '20'), multi=False) - variant('mpip_library', default=linux, + variant('mpip_library', default=False, description='Build stand-alone timemory-mpip GOTCHA library') - variant('ompt', default=True, description=('Enable OpenMP tools support')) - variant('ompt_standalone', default=True, + variant('ompt', default=False, description=('Enable OpenMP tools support')) + variant('ompt_standalone', default=False, description=('Enable OpenMP tools support via drop-in ' 'replacement of libomp/libgomp/libiomp5')) variant('ompt_llvm', default=False, description='Enable OpenMP tools support as part of llvm build') - variant('ompt_library', default=True, + variant('ompt_library', default=False, description='Build stand-alone timemory-ompt library') variant('allinea_map', default=False, description='Enable Allinea ARM-MAP support') - variant('require_packages', default=False, + variant('require_packages', default=True, description=('find_package(...) resulting in NOTFOUND ' 'generates error')) From 7717532d141522877347cf8d3037276f7bca85f1 Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 24 Jun 2020 22:12:48 -0500 Subject: [PATCH 09/38] The lapack-0.3.9-xerbl patch is no longer needed (#17217) The lapack-0.3.9-xerbl patch is not needed for versions 0.3.10 and above. --- var/spack/repos/builtin/packages/openblas/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 46cca3afe3c..c1a0c1e01bd 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -62,7 +62,7 @@ class Openblas(MakefilePackage): patch('openblas_icc_fortran.patch', when='%intel@16.0:') patch('openblas_icc_fortran2.patch', when='%intel@18.0:') # See https://github.com/spack/spack/issues/15385 - patch('lapack-0.3.9-xerbl.patch', when='@0.3.8: %intel') + patch('lapack-0.3.9-xerbl.patch', when='@0.3.8:0.3.9 %intel') # Fixes compilation error on POWER8 with GCC 7 # https://github.com/xianyi/OpenBLAS/pull/1098 From 30958649fb49a9600d2a9f9ed78b868d93cbde34 Mon Sep 17 00:00:00 2001 From: Chuck Atkins Date: Wed, 24 Jun 2020 23:18:49 -0400 Subject: [PATCH 10/38] ecp-io-sdk: remove deprecated numa variant from unify-fs (#17212) --- var/spack/repos/builtin/packages/ecp-io-sdk/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/ecp-io-sdk/package.py b/var/spack/repos/builtin/packages/ecp-io-sdk/package.py index ffc09974f12..a34ea99e275 100644 --- a/var/spack/repos/builtin/packages/ecp-io-sdk/package.py +++ b/var/spack/repos/builtin/packages/ecp-io-sdk/package.py @@ -35,7 +35,7 @@ class EcpIoSdk(CMakePackage): depends_on('hdf5+mpi+fortran', when='+hdf5') depends_on('mercury+mpi+ofi+sm', when='+mercury') depends_on('parallel-netcdf+fortran+pic', when='+pnetcdf') - depends_on('unifyfs+fortran+numa', when='+unifyfs') + depends_on('unifyfs+fortran', when='+unifyfs') depends_on('veloc', when='+veloc') def cmake_args(self): From eee56295ce5a637c44f382cda00ff9c881124926 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Wed, 24 Jun 2020 20:19:28 -0700 Subject: [PATCH 11/38] add cond py-mpi4py dep to conduit (#17211) --- var/spack/repos/builtin/packages/conduit/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py index 8481ddce82f..8996de7d9d4 100644 --- a/var/spack/repos/builtin/packages/conduit/package.py +++ b/var/spack/repos/builtin/packages/conduit/package.py @@ -93,6 +93,7 @@ class Conduit(Package): depends_on("python", when="+python") extends("python", when="+python") depends_on("py-numpy", when="+python", type=('build', 'run')) + depends_on("py-mpi4py", when="+python+mpi", type=('build', 'run')) ####################### # I/O Packages From a928f44fc0968aa57cec1bcd62cfbe1a0a267c32 Mon Sep 17 00:00:00 2001 From: cedricchevalier19 Date: Thu, 25 Jun 2020 05:22:40 +0200 Subject: [PATCH 12/38] Fix missing dependency in Mono (#17208) * Add missing dependency on Python for Mono. At build time * Adding version 6.8.0.123 for mono --- var/spack/repos/builtin/packages/mono/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/mono/package.py b/var/spack/repos/builtin/packages/mono/package.py index 6a7a96300ac..8b4e36b2b69 100644 --- a/var/spack/repos/builtin/packages/mono/package.py +++ b/var/spack/repos/builtin/packages/mono/package.py @@ -27,7 +27,10 @@ class Mono(AutotoolsPackage): depends_on('cmake~openssl', type=('build')) depends_on('iconv') depends_on('perl', type=('build')) + depends_on('python', type=('build')) + version('6.8.0.123', sha256='e2e42d36e19f083fc0d82f6c02f7db80611d69767112af353df2f279744a2ac5', + url='https://download.mono-project.com/sources/mono/mono-6.8.0.123.tar.xz') version('6.8.0.105', sha256='578799c44c3c86a9eb5daf6dec6c60a24341940fd376371956d4a46cf8612178', url='https://download.mono-project.com/sources/mono/mono-6.8.0.105.tar.xz') version('5.18.0.240', sha256='143e80eb00519ff496742e78ee07403a3c3629437f3a498eee539de8108da895') From e81533eb9bbd52384b6340cfaf1d354bf77cc058 Mon Sep 17 00:00:00 2001 From: oracleLee <52901865+oracleLee@users.noreply.github.com> Date: Thu, 25 Jun 2020 11:23:17 +0800 Subject: [PATCH 13/38] Update a supported package 'user-meamc' for lammps (#17207) * Update package.py * edit confliction when add package 'meam' The USER-MEAMC fully replaces the MEAM package, which has been removed from LAMMPS after the 12 December 2018 version. --- var/spack/repos/builtin/packages/lammps/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/lammps/package.py b/var/spack/repos/builtin/packages/lammps/package.py index b578016311a..928a7fddabf 100644 --- a/var/spack/repos/builtin/packages/lammps/package.py +++ b/var/spack/repos/builtin/packages/lammps/package.py @@ -61,8 +61,8 @@ def url_for_version(self, version): 'molecule', 'mpiio', 'peri', 'poems', 'python', 'qeq', 'replica', 'rigid', 'shock', 'snap', 'spin', 'srd', 'user-atc', 'user-h5md', 'user-lb', - 'user-misc', 'user-netcdf', 'user-omp', 'user-reaxc', - 'voronoi'] + 'user-meamc', 'user-misc', 'user-netcdf', 'user-omp', + 'user-reaxc', 'voronoi'] for pkg in supported_packages: variant(pkg, default=False, @@ -120,6 +120,8 @@ def url_for_version(self, version): conflicts('+user-misc', when='~manybody') conflicts('%gcc@9:', when='@:20200303+openmp') conflicts('+kokkos', when='@:20200227') + conflicts('+meam', when='@20181212:') + conflicts('+user-meamc', when='@:20181212') patch("lib.patch", when="@20170901") patch("660.patch", when="@20170922") From fe14c201dc3fb28a22808747cfddc43b0eb3efae Mon Sep 17 00:00:00 2001 From: Jen Herting Date: Wed, 24 Jun 2020 23:24:16 -0400 Subject: [PATCH 14/38] New package: py-mmtf-python (#17201) * mdanalysis * [py-mmtf-python] fixed copyright Co-authored-by: Andrew Elble --- .../packages/py-mmtf-python/package.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-mmtf-python/package.py diff --git a/var/spack/repos/builtin/packages/py-mmtf-python/package.py b/var/spack/repos/builtin/packages/py-mmtf-python/package.py new file mode 100644 index 00000000000..e86f4b7f8e2 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mmtf-python/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class PyMmtfPython(PythonPackage): + """The macromolecular transmission format (MMTF) is a binary encoding of + biological structures.""" + + homepage = "https://github.com/rcsb/mmtf-python" + url = "https://pypi.io/packages/source/m/mmtf-python/mmtf-python-1.1.2.tar.gz" + + version('1.1.2', sha256='a5caa7fcd2c1eaa16638b5b1da2d3276cbd3ed3513f0c2322957912003b6a8df') + + depends_on('py-setuptools', type='build') + depends_on('py-msgpack@0.5.6:', type=('build', 'run')) From b8270559d95ec584054c310973e3311e8d0b071d Mon Sep 17 00:00:00 2001 From: vvolkl Date: Thu, 25 Jun 2020 05:42:41 +0200 Subject: [PATCH 15/38] [podio] patch to correctly load dictionaries in v00-10 (#17157) --- .../builtin/packages/podio/dictloading.patch | 20 +++++++++++++++++++ .../repos/builtin/packages/podio/package.py | 1 + 2 files changed, 21 insertions(+) create mode 100644 var/spack/repos/builtin/packages/podio/dictloading.patch diff --git a/var/spack/repos/builtin/packages/podio/dictloading.patch b/var/spack/repos/builtin/packages/podio/dictloading.patch new file mode 100644 index 00000000000..15ec301b139 --- /dev/null +++ b/var/spack/repos/builtin/packages/podio/dictloading.patch @@ -0,0 +1,20 @@ +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 120a899..05991f1 100755 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -41,7 +41,7 @@ SET(headers + ${CMAKE_SOURCE_DIR}/include/podio/PythonEventStore.h + ) + PODIO_GENERATE_DICTIONARY(podioDict ${headers} SELECTION selection.xml +- OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}podio${CMAKE_SHARED_LIBRARY_SUFFIX} ++ OPTIONS --library ${CMAKE_SHARED_LIBRARY_PREFIX}podioDict${CMAKE_SHARED_LIBRARY_SUFFIX} + ) + # prevent generating dictionary twice + set_target_properties(podioDict-dictgen PROPERTIES EXCLUDE_FROM_ALL TRUE) +@@ -58,5 +58,5 @@ install(TARGETS podio podioDict podioRootIO + install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/podio DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/podioDictDict.rootmap +- ${CMAKE_CURRENT_BINARY_DIR}/libpodio_rdict.pcm ++ ${CMAKE_CURRENT_BINARY_DIR}/libpodioDict_rdict.pcm + DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/var/spack/repos/builtin/packages/podio/package.py b/var/spack/repos/builtin/packages/podio/package.py index 488eaf54f15..7211cdf1115 100644 --- a/var/spack/repos/builtin/packages/podio/package.py +++ b/var/spack/repos/builtin/packages/podio/package.py @@ -28,6 +28,7 @@ class Podio(CMakePackage): # cpack config throws an error on some systems patch('cpack.patch', when="@:0.10.0") + patch('dictloading.patch', when="@0.10.0") depends_on('root@6.08.06:') From 8d67279181f81f6053659a79f013b1f6266e3f70 Mon Sep 17 00:00:00 2001 From: Sergey Kosukhin Date: Thu, 25 Jun 2020 05:43:30 +0200 Subject: [PATCH 16/38] openjdk and jdk: extend the list of provided versions of java (#17151) --- var/spack/repos/builtin/packages/jdk/package.py | 1 + var/spack/repos/builtin/packages/openjdk/package.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index 92af25ae9b6..a90ddff9049 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -60,6 +60,7 @@ class Jdk(Package): url='https://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz') provides('java@14', when='@14.0:14.999') + provides('java@13', when='@13.0:13.999') provides('java@12', when='@12.0:12.999') provides('java@11', when='@11.0:11.999') provides('java@10', when='@10.0:10.999') diff --git a/var/spack/repos/builtin/packages/openjdk/package.py b/var/spack/repos/builtin/packages/openjdk/package.py index 1ad068936c6..9b520ded996 100644 --- a/var/spack/repos/builtin/packages/openjdk/package.py +++ b/var/spack/repos/builtin/packages/openjdk/package.py @@ -44,8 +44,10 @@ class Openjdk(Package): if pkg: version(ver, sha256=pkg[0], url=pkg[1]) - provides('java@8', when='@1.8.0:1.8.999') provides('java@11', when='@11.0:11.99') + provides('java@10', when='@10.0:10.99') + provides('java@9', when='@9.0:9.99') + provides('java@8', when='@1.8.0:1.8.999') conflicts('target=ppc64:', msg='openjdk is only available for x86_64 and aarch64') conflicts('target=ppc64le:', msg='openjdk is only available for x86_64 and aarch64') From cee24fbc9871095e9d817f2c16a3d74401f7b641 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 25 Jun 2020 05:49:30 +0200 Subject: [PATCH 17/38] mvapich2: Fix build with gcc@10: (#17114) --- var/spack/repos/builtin/packages/mvapich2/package.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index dac9b69c496..0e0ab26a22b 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -199,6 +199,16 @@ def file_system_options(self): return opts + def flag_handler(self, name, flags): + if name == 'fflags': + # https://bugzilla.redhat.com/show_bug.cgi?id=1795817 + if self.spec.satisfies('%gcc@10:'): + if flags is None: + flags = [] + flags.append('-fallow-argument-mismatch') + + return (flags, None, None) + def setup_build_environment(self, env): # mvapich2 configure fails when F90 and F90FLAGS are set env.unset('F90') From b2a4af764c5b4612426a73a611d1febee28f081c Mon Sep 17 00:00:00 2001 From: "Tomoki, Karatsu" <49965247+t-karatsu@users.noreply.github.com> Date: Thu, 25 Jun 2020 12:52:31 +0900 Subject: [PATCH 18/38] vtk: Add patch for finding Fujitsu-MPI wrapper commands. (#17069) --- .../packages/vtk/find_fujitsu_mpi.patch | 18 ++++++++++++++++++ .../repos/builtin/packages/vtk/package.py | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 var/spack/repos/builtin/packages/vtk/find_fujitsu_mpi.patch diff --git a/var/spack/repos/builtin/packages/vtk/find_fujitsu_mpi.patch b/var/spack/repos/builtin/packages/vtk/find_fujitsu_mpi.patch new file mode 100644 index 00000000000..c5ad2eb8300 --- /dev/null +++ b/var/spack/repos/builtin/packages/vtk/find_fujitsu_mpi.patch @@ -0,0 +1,18 @@ +--- VTK-8.2.0/CMake/FindMPI.cmake.org 2020-06-11 14:50:51.000000000 +0900 ++++ VTK-8.2.0/CMake/FindMPI.cmake 2020-06-11 14:51:21.000000000 +0900 +@@ -99,12 +99,12 @@ + # + + # Start out with the generic MPI compiler names, as these are most commonly used. +-set(_MPI_C_COMPILER_NAMES mpicc mpcc mpicc_r mpcc_r) ++set(_MPI_C_COMPILER_NAMES mpicc mpcc mpicc_r mpcc_r, mpifcc) + set(_MPI_CXX_COMPILER_NAMES mpicxx mpiCC mpcxx mpCC mpic++ mpc++ +- mpicxx_r mpiCC_r mpcxx_r mpCC_r mpic++_r mpc++_r) ++ mpicxx_r mpiCC_r mpcxx_r mpCC_r mpic++_r mpc++_r, mpiFCC) + set(_MPI_Fortran_COMPILER_NAMES mpif95 mpif95_r mpf95 mpf95_r + mpif90 mpif90_r mpf90 mpf90_r +- mpif77 mpif77_r mpf77 mpf77_r) ++ mpif77 mpif77_r mpf77 mpf77_r, mpifrt) + + # GNU compiler names + set(_MPI_GNU_C_COMPILER_NAMES mpigcc mpgcc mpigcc_r mpgcc_r) diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py index e99f779fede..0df4c4a5fca 100644 --- a/var/spack/repos/builtin/packages/vtk/package.py +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -103,6 +103,9 @@ class Vtk(CMakePackage): depends_on('double-conversion', when='@8.2.0:') depends_on('sqlite', when='@8.2.0:') + # For finding Fujitsu-MPI wrapper commands + patch('find_fujitsu_mpi.patch', when='%fj') + def url_for_version(self, version): url = "http://www.vtk.org/files/release/{0}/VTK-{1}.tar.gz" return url.format(version.up_to(2), version) From d8a7cfc36ae4168793912c151886d0797a15aa21 Mon Sep 17 00:00:00 2001 From: John Jolly Date: Wed, 24 Jun 2020 21:53:09 -0600 Subject: [PATCH 19/38] chill: Patch to include gmp, isl, and libquadmath (#16996) The rose library uses the `strtoflt128` and `quadmath_snprintf` functions. In order to successfully link the rose library, chill must also link the GCC libquadmath library to resolve the two functions. This patch changes the chill build to include this library. Chill will also not compile unless headers from the gmp and isl libraries are found in the includes path. Two patches - one each for gmp and isl - modify the chill build process to add options to specify those paths. These options follow the similar pattern as seen with BOOSTHOME and ROSEHOME options which already exist in the chill build process. Because of the addition of GMPHOME and ISLHOME options, build requirements for gmp and isl are also added. --- .../chill/Add-GCC-libquadmath-for-rose.patch | 25 +++++++++ .../packages/chill/Add-GMPHOME-option.patch | 53 +++++++++++++++++++ .../packages/chill/Add-ISLHOME-option.patch | 53 +++++++++++++++++++ .../repos/builtin/packages/chill/package.py | 20 ++++++- 4 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin/packages/chill/Add-GCC-libquadmath-for-rose.patch create mode 100644 var/spack/repos/builtin/packages/chill/Add-GMPHOME-option.patch create mode 100644 var/spack/repos/builtin/packages/chill/Add-ISLHOME-option.patch diff --git a/var/spack/repos/builtin/packages/chill/Add-GCC-libquadmath-for-rose.patch b/var/spack/repos/builtin/packages/chill/Add-GCC-libquadmath-for-rose.patch new file mode 100644 index 00000000000..68b335097fb --- /dev/null +++ b/var/spack/repos/builtin/packages/chill/Add-GCC-libquadmath-for-rose.patch @@ -0,0 +1,25 @@ +From 15c3cc127d8f48324b00fd654d9630724eedcfcd Mon Sep 17 00:00:00 2001 +From: John Jolly +Date: Fri, 5 Jun 2020 23:54:13 -0600 +Subject: [PATCH] Add GCC libquadmath for rose + +--- + Makefile.am | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile.am b/Makefile.am +index 7dd92b0..a8daba1 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -7,7 +7,7 @@ core_cxxflags =-g --std=c++11 + + ## Core Libraries ## + core_libs = -lcodegen -lomega +-core_libs += -lm -lrose -lrt -lutil -ldl ++core_libs += -lm -lrose -lrt -lutil -ldl -lquadmath + core_libs += -lboost_date_time -lboost_filesystem -lboost_program_options + core_libs += -lboost_regex -lboost_system -lboost_wave -lboost_iostreams + +-- +2.25.1 + diff --git a/var/spack/repos/builtin/packages/chill/Add-GMPHOME-option.patch b/var/spack/repos/builtin/packages/chill/Add-GMPHOME-option.patch new file mode 100644 index 00000000000..ba6b19693a9 --- /dev/null +++ b/var/spack/repos/builtin/packages/chill/Add-GMPHOME-option.patch @@ -0,0 +1,53 @@ +From 336dfb3a1c314b6eec23ab4f99114be94c5e518e Mon Sep 17 00:00:00 2001 +From: John Jolly +Date: Fri, 5 Jun 2020 23:45:39 -0600 +Subject: [PATCH] Add GMPHOME option + +--- + Makefile.am | 4 ++++ + configure.ac | 7 +++++++ + 2 files changed, 11 insertions(+) + +diff --git a/Makefile.am b/Makefile.am +index 541cb99..7dd92b0 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -16,6 +16,8 @@ core_libs += -liegenlib -lisl -lgmp + core_libdirs = -Lomega/code_gen/obj -Lomega/omega_lib/obj + core_libdirs += -L$(ROSEHOME)/lib -L$(BOOSTHOME)/lib + ++core_libdirs += -L$(GMPHOME)/lib ++ + core_libdirs += -L$(ISLHOME)/lib + + core_libdirs += -L$(IEGENHOME)/lib +@@ -32,6 +34,8 @@ core_includes += -I$(srcdir)/include/chill + core_includes += -I$(ROSEHOME)/include/rose + core_includes += -I$(BOOSTHOME)/include + ++core_includes += -I$(GSLHOME)/include ++ + core_includes += -I$(ISLHOME)/include + + core_includes += -I$(IEGENHOME)/include/iegenlib +diff --git a/configure.ac b/configure.ac +index e146839..18ecf76 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -58,6 +58,13 @@ AC_ARG_WITH([boost],[ + ],[ + AC_SUBST([BOOSTHOME], ["${BOOSTHOME}"])]) + ++AC_ARG_WITH([gmp],[ ++ AS_HELP_STRING([--with-gmp],[set path to gmp]) ++ ],[ ++ AC_SUBST([GMPHOME], [$withval]) ++ ],[ ++ AC_SUBST([GMPHOME], ["${GMPHOME}"])]) ++ + AC_ARG_WITH([iegen],[ + AS_HELP_STRING([--with-iegen],[set path to iegenlib]) + ],[ +-- +2.25.1 + diff --git a/var/spack/repos/builtin/packages/chill/Add-ISLHOME-option.patch b/var/spack/repos/builtin/packages/chill/Add-ISLHOME-option.patch new file mode 100644 index 00000000000..e15456cd576 --- /dev/null +++ b/var/spack/repos/builtin/packages/chill/Add-ISLHOME-option.patch @@ -0,0 +1,53 @@ +From 3b4094b9cf1beec0b870e9ca0c3e51ed9b10dc39 Mon Sep 17 00:00:00 2001 +From: John Jolly +Date: Fri, 5 Jun 2020 23:21:18 -0600 +Subject: [PATCH] Add ISLHOME option + +--- + Makefile.am | 4 ++++ + configure.ac | 7 +++++++ + 2 files changed, 11 insertions(+) + +diff --git a/Makefile.am b/Makefile.am +index 30667a8..541cb99 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -16,6 +16,8 @@ core_libs += -liegenlib -lisl -lgmp + core_libdirs = -Lomega/code_gen/obj -Lomega/omega_lib/obj + core_libdirs += -L$(ROSEHOME)/lib -L$(BOOSTHOME)/lib + ++core_libdirs += -L$(ISLHOME)/lib ++ + core_libdirs += -L$(IEGENHOME)/lib + core_libdirs += -L$(IEGENHOME)/../lib/installed/lib + +@@ -30,6 +32,8 @@ core_includes += -I$(srcdir)/include/chill + core_includes += -I$(ROSEHOME)/include/rose + core_includes += -I$(BOOSTHOME)/include + ++core_includes += -I$(ISLHOME)/include ++ + core_includes += -I$(IEGENHOME)/include/iegenlib + core_includes += -I$(IEGENHOME)/../lib/installed/include + +diff --git a/configure.ac b/configure.ac +index 8e163cd..e146839 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -65,6 +65,13 @@ AC_ARG_WITH([iegen],[ + ],[ + AC_SUBST([IEGENHOME], ["${IEGENHOME}"])]) + ++AC_ARG_WITH([isl],[ ++ AS_HELP_STRING([--with-isl],[set path to isl]) ++ ],[ ++ AC_SUBST([ISLHOME], [$withval]) ++ ],[ ++ AC_SUBST([ISLHOME], ["${ISLHOME}"])]) ++ + AC_ARG_WITH([omega],[ + AS_HELP_STRING([--with-omega],[set omega home])],[ + AC_SUBST([OMEGAHOME], [$withval])],[ +-- +2.25.1 + diff --git a/var/spack/repos/builtin/packages/chill/package.py b/var/spack/repos/builtin/packages/chill/package.py index 96afa55379b..1358453b09f 100644 --- a/var/spack/repos/builtin/packages/chill/package.py +++ b/var/spack/repos/builtin/packages/chill/package.py @@ -29,6 +29,12 @@ class Chill(AutotoolsPackage): depends_on('flex', type='build') # Does not currrently work with Python3 depends_on('python@2.7:2.8') + depends_on('isl', type='build') + depends_on('gmp', type='build') + + patch('Add-ISLHOME-option.patch') + patch('Add-GMPHOME-option.patch') + patch('Add-GCC-libquadmath-for-rose.patch') build_directory = 'spack-build' @@ -40,27 +46,39 @@ def setup_build_environment(self, env): rose_home = self.spec['rose'].prefix boost_home = self.spec['boost'].prefix iegen_home = self.spec['iegenlib'].prefix + isl_home = self.spec['isl'].prefix + gmp_home = self.spec['gmp'].prefix env.set('ROSEHOME', rose_home) env.set('BOOSTHOME', boost_home) env.set('IEGENHOME', iegen_home) + env.set('ISLHOME', isl_home) + env.set('GMPHOME', gmp_home) env.append_path('LD_LIBRARY_PATH', rose_home.lib) env.append_path('LD_LIBRARY_PATH', boost_home.lib) env.append_path('LD_LIBRARY_PATH', iegen_home.lib) + env.append_path('LD_LIBRARY_PATH', isl_home.lib) + env.append_path('LD_LIBRARY_PATH', gmp_home.lib) def setup_run_environment(self, env): rose_home = self.spec['rose'].prefix boost_home = self.spec['boost'].prefix iegen_home = self.spec['iegenlib'].prefix + isl_home = self.spec['isl'].prefix + gmp_home = self.spec['gmp'].prefix env.append_path('LD_LIBRARY_PATH', rose_home.lib) env.append_path('LD_LIBRARY_PATH', boost_home.lib) env.append_path('LD_LIBRARY_PATH', iegen_home.lib) + env.append_path('LD_LIBRARY_PATH', isl_home.lib) + env.append_path('LD_LIBRARY_PATH', gmp_home.lib) def configure_args(self): args = ['--with-rose={0}'.format(self.spec['rose'].prefix), '--with-boost={0}'.format(self.spec['boost'].prefix), - '--with-iegen={0}'.format(self.spec['iegenlib'].prefix)] + '--with-iegen={0}'.format(self.spec['iegenlib'].prefix), + '--with-isl={0}'.format(self.spec['isl'].prefix), + '--with-gmp={0}'.format(self.spec['gmp'].prefix)] return args From 089a21dd1db017a1b20e661f46b85b63beb9ce8f Mon Sep 17 00:00:00 2001 From: Brian Van Essen Date: Wed, 24 Jun 2020 21:00:06 -0700 Subject: [PATCH 20/38] Update the version of Cereal. (#17244) --- var/spack/repos/builtin/packages/cereal/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cereal/package.py b/var/spack/repos/builtin/packages/cereal/package.py index ddab8916542..51503b2fbeb 100644 --- a/var/spack/repos/builtin/packages/cereal/package.py +++ b/var/spack/repos/builtin/packages/cereal/package.py @@ -18,6 +18,7 @@ class Cereal(CMakePackage): homepage = "http://uscilab.github.io/cereal/" url = "https://github.com/USCiLab/cereal/archive/v1.1.2.tar.gz" + version('1.3.0', sha256='329ea3e3130b026c03a4acc50e168e7daff4e6e661bc6a7dfec0d77b570851d5') version('1.2.2', sha256='1921f26d2e1daf9132da3c432e2fd02093ecaedf846e65d7679ddf868c7289c4') version('1.2.1', sha256='7d321c22ea1280b47ddb06f3e9702fcdbb2910ff2f3df0a2554804210714434e') version('1.2.0', sha256='1ccf3ed205a7a2f0d6a060415b123f1ae0d984cd4435db01af8de11a2eda49c1') From b26e93af3d19e4492cf049bc7479eb85bc8f7da7 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 25 Jun 2020 02:38:01 -0500 Subject: [PATCH 21/38] spack config: new subcommands add/remove (#13920) spack config add : add nested value value to the configuration scope specified spack config remove/rm: remove specified configuration from the relevant scope --- lib/spack/spack/cmd/config.py | 169 +++++++++- lib/spack/spack/cmd/external.py | 2 +- lib/spack/spack/config.py | 213 +++++++++--- lib/spack/spack/environment.py | 86 +++-- lib/spack/spack/schema/env.py | 2 + lib/spack/spack/test/cmd/config.py | 310 +++++++++++++++++- lib/spack/spack/test/config.py | 2 +- lib/spack/spack/test/conftest.py | 26 ++ lib/spack/spack/test/container/cli.py | 4 +- lib/spack/spack/test/container/conftest.py | 2 +- lib/spack/spack/test/container/docker.py | 2 +- lib/spack/spack/test/container/singularity.py | 2 +- share/spack/spack-completion.bash | 29 +- 13 files changed, 746 insertions(+), 103 deletions(-) diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index b1a6454555d..954d4e4585e 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -5,12 +5,14 @@ from __future__ import print_function import os +import re import llnl.util.tty as tty import spack.config +import spack.schema.env import spack.environment as ev - +import spack.util.spack_yaml as syaml from spack.util.editor import editor description = "get and set configuration options" @@ -58,24 +60,48 @@ def setup_parser(subparser): sp.add_parser('list', help='list configuration sections') + add_parser = sp.add_parser('add', help='add configuration parameters') + add_parser.add_argument( + 'path', nargs='?', + help="colon-separated path to config that should be added," + " e.g. 'config:default:true'") + add_parser.add_argument( + '-f', '--file', + help="file from which to set all config values" + ) + + remove_parser = sp.add_parser('remove', aliases=['rm'], + help='remove configuration parameters') + remove_parser.add_argument( + 'path', + help="colon-separated path to config that should be removed," + " e.g. 'config:default:true'") + + # Make the add parser available later + setup_parser.add_parser = add_parser + def _get_scope_and_section(args): """Extract config scope and section from arguments.""" scope = args.scope - section = args.section + section = getattr(args, 'section', None) + path = getattr(args, 'path', None) # w/no args and an active environment, point to env manifest - if not args.section: + if not section: env = ev.get_env(args, 'config edit') if env: scope = env.env_file_config_scope_name() # set scope defaults - elif not args.scope: - if section == 'compilers': - scope = spack.config.default_modify_scope() - else: - scope = 'user' + elif not scope: + scope = spack.config.default_modify_scope(section) + + # special handling for commands that take value instead of section + if path: + section = path[:path.find(':')] if ':' in path else path + if not scope: + scope = spack.config.default_modify_scope(section) return scope, section @@ -135,11 +161,126 @@ def config_list(args): print(' '.join(list(spack.config.section_schemas))) +def set_config(args, section, new, scope): + if re.match(r'env.*', scope): + e = ev.get_env(args, 'config add') + e.set_config(section, new) + else: + spack.config.set(section, new, scope=scope) + + +def config_add(args): + """Add the given configuration to the specified config scope + + This is a stateful operation that edits the config files.""" + if not (args.file or args.path): + tty.error("No changes requested. Specify a file or value.") + setup_parser.add_parser.print_help() + exit(1) + + scope, section = _get_scope_and_section(args) + + # Updates from file + if args.file: + # Get file as config dict + data = spack.config.read_config_file(args.file) + if any(k in data for k in spack.schema.env.keys): + data = ev.config_dict(data) + + # update all sections from config dict + # We have to iterate on keys to keep overrides from the file + for section in data.keys(): + if section in spack.config.section_schemas.keys(): + # Special handling for compiler scope difference + # Has to be handled after we choose a section + if scope is None: + scope = spack.config.default_modify_scope(section) + + value = data[section] + existing = spack.config.get(section, scope=scope) + new = spack.config.merge_yaml(existing, value) + + set_config(args, section, new, scope) + + if args.path: + components = spack.config.process_config_path(args.path) + + has_existing_value = True + path = '' + override = False + for idx, name in enumerate(components[:-1]): + # First handle double colons in constructing path + colon = '::' if override else ':' if path else '' + path += colon + name + if getattr(name, 'override', False): + override = True + else: + override = False + + # Test whether there is an existing value at this level + existing = spack.config.get(path, scope=scope) + + if existing is None: + has_existing_value = False + # We've nested further than existing config, so we need the + # type information for validation to know how to handle bare + # values appended to lists. + existing = spack.config.get_valid_type(path) + + # construct value from this point down + value = syaml.load_config(components[-1]) + for component in reversed(components[idx + 1:-1]): + value = {component: value} + break + + if has_existing_value: + path, _, value = args.path.rpartition(':') + value = syaml.load_config(value) + existing = spack.config.get(path, scope=scope) + + # append values to lists + if isinstance(existing, list) and not isinstance(value, list): + value = [value] + + # merge value into existing + new = spack.config.merge_yaml(existing, value) + set_config(args, path, new, scope) + + +def config_remove(args): + """Remove the given configuration from the specified config scope + + This is a stateful operation that edits the config files.""" + scope, _ = _get_scope_and_section(args) + + path, _, value = args.path.rpartition(':') + existing = spack.config.get(path, scope=scope) + + if not isinstance(existing, (list, dict)): + path, _, value = path.rpartition(':') + existing = spack.config.get(path, scope=scope) + + value = syaml.load(value) + + if isinstance(existing, list): + values = value if isinstance(value, list) else [value] + for v in values: + existing.remove(v) + elif isinstance(existing, dict): + existing.pop(value, None) + else: + # This should be impossible to reach + raise spack.config.ConfigError('Config has nested non-dict values') + + set_config(args, path, existing, scope) + + def config(parser, args): - action = { - 'get': config_get, - 'blame': config_blame, - 'edit': config_edit, - 'list': config_list, - } + action = {'get': config_get, + 'blame': config_blame, + 'edit': config_edit, + 'list': config_list, + 'add': config_add, + 'rm': config_remove, + 'remove': config_remove} action[args.config_command](args) diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index f93deaba037..afdd40e2a04 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -186,7 +186,7 @@ def _update_pkg_config(pkg_to_entries, not_buildable): cfg_scope = spack.config.default_modify_scope() pkgs_cfg = spack.config.get('packages', scope=cfg_scope) - spack.config._merge_yaml(pkgs_cfg, pkg_to_cfg) + spack.config.merge_yaml(pkgs_cfg, pkg_to_cfg) spack.config.set('packages', pkgs_cfg, scope=cfg_scope) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 445d62d2abc..a3d8101cad6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -56,12 +56,12 @@ import spack.schema.modules import spack.schema.config import spack.schema.upstreams +import spack.schema.env from spack.error import SpackError # Hacked yaml for configuration files preserves line numbers. import spack.util.spack_yaml as syaml - #: Dict from section names -> schema for that section section_schemas = { 'compilers': spack.schema.compilers.schema, @@ -70,9 +70,15 @@ 'packages': spack.schema.packages.schema, 'modules': spack.schema.modules.schema, 'config': spack.schema.config.schema, - 'upstreams': spack.schema.upstreams.schema + 'upstreams': spack.schema.upstreams.schema, } +# Same as above, but including keys for environments +# this allows us to unify config reading between configs and environments +all_schemas = copy.deepcopy(section_schemas) +all_schemas.update(dict((key, spack.schema.env.schema) + for key in spack.schema.env.keys)) + #: Builtin paths to configuration files in Spack configuration_paths = ( # Default configuration scope is the lowest-level scope. These are @@ -142,19 +148,21 @@ def get_section(self, section): if section not in self.sections: path = self.get_section_filename(section) schema = section_schemas[section] - data = _read_config_file(path, schema) + data = read_config_file(path, schema) self.sections[section] = data return self.sections[section] def write_section(self, section): filename = self.get_section_filename(section) data = self.get_section(section) - validate(data, section_schemas[section]) + + # We copy data here to avoid adding defaults at write time + validate_data = copy.deepcopy(data) + validate(validate_data, section_schemas[section]) try: mkdirp(self.path) with open(filename, 'w') as f: - validate(data, section_schemas[section]) syaml.dump_config(data, stream=f, default_flow_style=False) except (yaml.YAMLError, IOError) as e: raise ConfigFileError( @@ -217,7 +225,7 @@ def get_section(self, section): # } # } if self._raw_data is None: - self._raw_data = _read_config_file(self.path, self.schema) + self._raw_data = read_config_file(self.path, self.schema) if self._raw_data is None: return None @@ -376,6 +384,16 @@ def highest_precedence_scope(self): """Non-internal scope with highest precedence.""" return next(reversed(self.file_scopes), None) + def highest_precedence_non_platform_scope(self): + """Non-internal non-platform scope with highest precedence + + Platform-specific scopes are of the form scope/platform""" + generator = reversed(self.file_scopes) + highest = next(generator, None) + while highest and '/' in highest.name: + highest = next(generator, None) + return highest + def matching_scopes(self, reg_expr): """ List of all scopes whose names match the provided regular expression. @@ -435,8 +453,21 @@ def update_config(self, section, update_data, scope=None): _validate_section_name(section) # validate section name scope = self._validate_scope(scope) # get ConfigScope object + # manually preserve comments + need_comment_copy = (section in scope.sections and + scope.sections[section] is not None) + if need_comment_copy: + comments = getattr(scope.sections[section][section], + yaml.comments.Comment.attrib, + None) + # read only the requested section's data. - scope.sections[section] = {section: update_data} + scope.sections[section] = syaml.syaml_dict({section: update_data}) + if need_comment_copy and comments: + setattr(scope.sections[section][section], + yaml.comments.Comment.attrib, + comments) + scope.write_section(section) def get_config(self, section, scope=None): @@ -483,14 +514,17 @@ def get_config(self, section, scope=None): if section not in data: continue - merged_section = _merge_yaml(merged_section, data) + merged_section = merge_yaml(merged_section, data) # no config files -- empty config. if section not in merged_section: - return {} + return syaml.syaml_dict() # take the top key off before returning. - return merged_section[section] + ret = merged_section[section] + if isinstance(ret, dict): + ret = syaml.syaml_dict(ret) + return ret def get(self, path, default=None, scope=None): """Get a config section or a single value from one. @@ -506,14 +540,12 @@ def get(self, path, default=None, scope=None): We use ``:`` as the separator, like YAML objects. """ - # TODO: Currently only handles maps. Think about lists if neded. - section, _, rest = path.partition(':') + # TODO: Currently only handles maps. Think about lists if needed. + parts = process_config_path(path) + section = parts.pop(0) value = self.get_config(section, scope=scope) - if not rest: - return value - parts = rest.split(':') while parts: key = parts.pop(0) value = value.get(key, default) @@ -525,21 +557,40 @@ def set(self, path, value, scope=None): Accepts the path syntax described in ``get()``. """ - parts = _process_config_path(path) + if ':' not in path: + # handle bare section name as path + self.update_config(path, value, scope=scope) + return + + parts = process_config_path(path) section = parts.pop(0) - if not parts: - self.update_config(section, value, scope=scope) - else: - section_data = self.get_config(section, scope=scope) + section_data = self.get_config(section, scope=scope) - data = section_data - while len(parts) > 1: - key = parts.pop(0) - data = data[key] - data[parts[0]] = value + data = section_data + while len(parts) > 1: + key = parts.pop(0) - self.update_config(section, section_data, scope=scope) + if _override(key): + new = type(data[key])() + del data[key] + else: + new = data[key] + + if isinstance(new, dict): + # Make it an ordered dict + new = syaml.syaml_dict(new) + # reattach to parent object + data[key] = new + data = new + + if _override(parts[0]): + data.pop(parts[0], None) + + # update new value + data[parts[0]] = value + + self.update_config(section, section_data, scope=scope) def __iter__(self): """Iterate over scopes in this configuration.""" @@ -692,26 +743,53 @@ def _validate_section_name(section): % (section, " ".join(section_schemas.keys()))) -def validate(data, schema, set_defaults=True): +def validate(data, schema, filename=None): """Validate data read in from a Spack YAML file. Arguments: data (dict or list): data read from a Spack YAML file schema (dict or list): jsonschema to validate data - set_defaults (bool): whether to set defaults based on the schema This leverages the line information (start_mark, end_mark) stored on Spack YAML structures. """ import jsonschema + # validate a copy to avoid adding defaults + # This allows us to round-trip data without adding to it. + test_data = copy.deepcopy(data) + + if isinstance(test_data, yaml.comments.CommentedMap): + # HACK to fully copy ruamel CommentedMap that doesn't provide copy + # method. Especially necessary for environments + setattr(test_data, + yaml.comments.Comment.attrib, + getattr(data, + yaml.comments.Comment.attrib, + yaml.comments.Comment())) + try: - spack.schema.Validator(schema).validate(data) + spack.schema.Validator(schema).validate(test_data) except jsonschema.ValidationError as e: - raise ConfigFormatError(e, data) + if hasattr(e.instance, 'lc'): + line_number = e.instance.lc.line + 1 + else: + line_number = None + raise ConfigFormatError(e, data, filename, line_number) + # return the validated data so that we can access the raw data + # mostly relevant for environments + return test_data -def _read_config_file(filename, schema): - """Read a YAML configuration file.""" +def read_config_file(filename, schema=None): + """Read a YAML configuration file. + + User can provide a schema for validation. If no schema is provided, + we will infer the schema from the top-level key.""" + # Dev: Inferring schema and allowing it to be provided directly allows us + # to preserve flexibility in calling convention (don't need to provide + # schema when it's not necessary) while allowing us to validate against a + # known schema when the top-level key could be incorrect. + # Ignore nonexisting files. if not os.path.exists(filename): return None @@ -729,9 +807,16 @@ def _read_config_file(filename, schema): data = syaml.load_config(f) if data: + if not schema: + key = next(iter(data)) + schema = all_schemas[key] validate(data, schema) return data + except StopIteration: + raise ConfigFileError( + "Config file is empty or is not a valid YAML dict: %s" % filename) + except MarkedYAMLError as e: raise ConfigFileError( "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) @@ -772,13 +857,40 @@ def _mark_internal(data, name): return d -def _merge_yaml(dest, source): +def get_valid_type(path): + """Returns an instance of a type that will pass validation for path. + + The instance is created by calling the constructor with no arguments. + If multiple types will satisfy validation for data at the configuration + path given, the priority order is ``list``, ``dict``, ``str``, ``bool``, + ``int``, ``float``. + """ + components = process_config_path(path) + section = components[0] + for type in (list, syaml.syaml_dict, str, bool, int, float): + try: + ret = type() + test_data = ret + for component in reversed(components): + test_data = {component: test_data} + validate(test_data, section_schemas[section]) + return ret + except (ConfigFormatError, AttributeError): + # This type won't validate, try the next one + # Except AttributeError because undefined behavior of dict ordering + # in python 3.5 can cause the validator to raise an AttributeError + # instead of a ConfigFormatError. + pass + raise ConfigError("Cannot determine valid type for path '%s'." % path) + + +def merge_yaml(dest, source): """Merges source into dest; entries in source take precedence over dest. This routine may modify dest and should be assigned to dest, in case dest was None to begin with, e.g.: - dest = _merge_yaml(dest, source) + dest = merge_yaml(dest, source) Config file authors can optionally end any attribute in a dict with `::` instead of `:`, and the key will override that of the @@ -793,6 +905,7 @@ def they_are(t): # Source list is prepended (for precedence) if they_are(list): + # Make sure to copy ruamel comments dest[:] = source + [x for x in dest if x not in source] return dest @@ -805,9 +918,10 @@ def they_are(t): if _override(sk) or sk not in dest: # if sk ended with ::, or if it's new, completely override dest[sk] = copy.copy(sv) + # copy ruamel comments manually else: # otherwise, merge the YAML - dest[sk] = _merge_yaml(dest[sk], source[sk]) + dest[sk] = merge_yaml(dest[sk], source[sk]) # this seems unintuitive, but see below. We need this because # Python dicts do not overwrite keys on insert, and we want @@ -837,7 +951,7 @@ def they_are(t): # Process a path argument to config.set() that may contain overrides ('::' or # trailing ':') # -def _process_config_path(path): +def process_config_path(path): result = [] if path.startswith(':'): raise syaml.SpackYAMLError("Illegal leading `:' in path `{0}'". @@ -861,13 +975,20 @@ def _process_config_path(path): # # Settings for commands that modify configuration # -def default_modify_scope(): +def default_modify_scope(section='config'): """Return the config scope that commands should modify by default. Commands that modify configuration by default modify the *highest* priority scope. + + Arguments: + section (boolean): Section for which to get the default scope. + If this is not 'compilers', a general (non-platform) scope is used. """ - return spack.config.config.highest_precedence_scope().name + if section == 'compilers': + return spack.config.config.highest_precedence_scope().name + else: + return spack.config.config.highest_precedence_non_platform_scope().name def default_list_scope(): @@ -894,17 +1015,17 @@ class ConfigFormatError(ConfigError): """Raised when a configuration format does not match its schema.""" def __init__(self, validation_error, data, filename=None, line=None): + # spack yaml has its own file/line marks -- try to find them + # we prioritize these over the inputs + mark = self._get_mark(validation_error, data) + if mark: + filename = mark.name + line = mark.line + 1 + self.filename = filename # record this for ruamel.yaml + # construct location location = '' - - # spack yaml has its own file/line marks -- try to find them - if not filename and not line: - mark = self._get_mark(validation_error, data) - if mark: - filename = mark.name - line = mark.line + 1 - if filename: location = '%s' % filename if line is not None: diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 1b9df358f47..bc9730e4908 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -80,9 +80,6 @@ #: version of the lockfile format. Must increase monotonically. lockfile_format_version = 2 -#: legal first keys in the spack.yaml manifest file -env_schema_keys = ('spack', 'env') - # Magic names # The name of the standalone spec list in the manifest yaml user_speclist_name = 'specs' @@ -366,7 +363,7 @@ def create(name, init_file=None, with_view=None): def config_dict(yaml_data): """Get the configuration scope section out of an spack.yaml""" - key = spack.config.first_existing(yaml_data, env_schema_keys) + key = spack.config.first_existing(yaml_data, spack.schema.env.keys) return yaml_data[key] @@ -392,42 +389,19 @@ def all_environments(): yield read(name) -def validate(data, filename=None): - # validating changes data by adding defaults. Return validated data - validate_data = copy.deepcopy(data) - # HACK to fully copy ruamel CommentedMap that doesn't provide copy method - import ruamel.yaml as yaml - setattr( - validate_data, - yaml.comments.Comment.attrib, - getattr(data, yaml.comments.Comment.attrib, yaml.comments.Comment()) - ) - - import jsonschema - try: - spack.schema.Validator(spack.schema.env.schema).validate(validate_data) - except jsonschema.ValidationError as e: - if hasattr(e.instance, 'lc'): - line_number = e.instance.lc.line + 1 - else: - line_number = None - raise spack.config.ConfigFormatError( - e, data, filename, line_number) - return validate_data - - def _read_yaml(str_or_file): """Read YAML from a file for round-trip parsing.""" data = syaml.load_config(str_or_file) filename = getattr(str_or_file, 'name', None) - default_data = validate(data, filename) + default_data = spack.config.validate( + data, spack.schema.env.schema, filename) return (data, default_data) def _write_yaml(data, str_or_file): """Write YAML to a file preserving comments and dict order.""" filename = getattr(str_or_file, 'name', None) - validate(data, filename) + spack.config.validate(data, spack.schema.env.schema, filename) syaml.dump_config(data, str_or_file, default_flow_style=False) @@ -815,12 +789,21 @@ def env_file_config_scope(self): return spack.config.SingleFileScope(config_name, self.manifest_path, spack.schema.env.schema, - [env_schema_keys]) + [spack.schema.env.keys]) def config_scopes(self): """A list of all configuration scopes for this environment.""" return self.included_config_scopes() + [self.env_file_config_scope()] + def set_config(self, path, value): + """Set configuration for this environment""" + yaml = config_dict(self.yaml) + keys = spack.config.process_config_path(path) + for key in keys[:-1]: + yaml = yaml[key] + yaml[keys[-1]] = value + self.write() + def destroy(self): """Remove this environment from Spack entirely.""" shutil.rmtree(self.path) @@ -1501,8 +1484,12 @@ def write(self, regenerate_views=True): del yaml_dict[key] # if all that worked, write out the manifest file at the top level - # Only actually write if it has changed or was never written - changed = self.yaml != self.raw_yaml + # (we used to check whether the yaml had changed and not write it out + # if it hadn't. We can't do that anymore because it could be the only + # thing that changed is the "override" attribute on a config dict, + # which would not show up in even a string comparison between the two + # keys). + changed = not yaml_equivalent(self.yaml, self.raw_yaml) written = os.path.exists(self.manifest_path) if changed or not written: self.raw_yaml = copy.deepcopy(self.yaml) @@ -1529,6 +1516,39 @@ def __exit__(self, exc_type, exc_val, exc_tb): activate(self._previous_active) +def yaml_equivalent(first, second): + """Returns whether two spack yaml items are equivalent, including overrides + """ + if isinstance(first, dict): + return isinstance(second, dict) and _equiv_dict(first, second) + elif isinstance(first, list): + return isinstance(second, list) and _equiv_list(first, second) + else: # it's a string + return isinstance(second, six.string_types) and first == second + + +def _equiv_list(first, second): + """Returns whether two spack yaml lists are equivalent, including overrides + """ + if len(first) != len(second): + return False + return all(yaml_equivalent(f, s) for f, s in zip(first, second)) + + +def _equiv_dict(first, second): + """Returns whether two spack yaml dicts are equivalent, including overrides + """ + if len(first) != len(second): + return False + same_values = all(yaml_equivalent(fv, sv) + for fv, sv in zip(first.values(), second.values())) + same_keys_with_same_overrides = all( + fk == sk and getattr(fk, 'override', False) == getattr(sk, 'override', + False) + for fk, sk in zip(first.keys(), second.keys())) + return same_values and same_keys_with_same_overrides + + def display_specs(concretized_specs): """Displays the list of specs returned by `Environment.concretize()`. diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 907abc5c9a2..6ead76416b6 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -13,6 +13,8 @@ import spack.schema.merged import spack.schema.projections +#: legal first keys in the schema +keys = ('spack', 'env') spec_list_schema = { 'type': 'array', diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 82a9d814ea9..6dbf50676db 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -2,7 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +import pytest import os from llnl.util.filesystem import mkdirp @@ -12,6 +12,7 @@ from spack.main import SpackCommand config = SpackCommand('config') +env = SpackCommand('env') def test_get_config_scope(mock_low_high_config): @@ -46,7 +47,7 @@ def test_get_config_scope_merged(mock_low_high_config): def test_config_edit(): """Ensure `spack config edit` edits the right paths.""" - dms = spack.config.default_modify_scope() + dms = spack.config.default_modify_scope('compilers') dms_path = spack.config.config.scopes[dms].path user_path = spack.config.config.scopes['user'].path @@ -97,3 +98,308 @@ def test_config_list(): output = config('list') assert 'compilers' in output assert 'packages' in output + + +def test_config_add(mutable_empty_config): + config('add', 'config:dirty:true') + output = config('get', 'config') + + assert output == """config: + dirty: true +""" + + +def test_config_add_list(mutable_empty_config): + config('add', 'config:template_dirs:test1') + config('add', 'config:template_dirs:[test2]') + config('add', 'config:template_dirs:test3') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test3 + - test2 + - test1 +""" + + +def test_config_add_override(mutable_empty_config): + config('--scope', 'site', 'add', 'config:template_dirs:test1') + config('add', 'config:template_dirs:[test2]') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test2 + - test1 +""" + + config('add', 'config::template_dirs:[test2]') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test2 +""" + + +def test_config_add_override_leaf(mutable_empty_config): + config('--scope', 'site', 'add', 'config:template_dirs:test1') + config('add', 'config:template_dirs:[test2]') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test2 + - test1 +""" + + config('add', 'config:template_dirs::[test2]') + output = config('get', 'config') + + assert output == """config: + 'template_dirs:': + - test2 +""" + + +def test_config_add_update_dict(mutable_empty_config): + config('add', 'packages:all:compiler:[gcc]') + config('add', 'packages:all:version:1.0.0') + output = config('get', 'packages') + + expected = """packages: + all: + compiler: [gcc] + version: + - 1.0.0 +""" + + assert output == expected + + +def test_config_add_ordered_dict(mutable_empty_config): + config('add', 'mirrors:first:/path/to/first') + config('add', 'mirrors:second:/path/to/second') + output = config('get', 'mirrors') + + assert output == """mirrors: + first: /path/to/first + second: /path/to/second +""" + + +def test_config_add_invalid_fails(mutable_empty_config): + config('add', 'packages:all:variants:+debug') + with pytest.raises( + (spack.config.ConfigFormatError, AttributeError) + ): + config('add', 'packages:all:True') + + +def test_config_add_from_file(mutable_empty_config, tmpdir): + contents = """spack: + config: + dirty: true +""" + + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + config('add', '-f', file) + output = config('get', 'config') + + assert output == """config: + dirty: true +""" + + +def test_config_add_from_file_multiple(mutable_empty_config, tmpdir): + contents = """spack: + config: + dirty: true + template_dirs: [test1] +""" + + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + config('add', '-f', file) + output = config('get', 'config') + + assert output == """config: + dirty: true + template_dirs: [test1] +""" + + +def test_config_add_override_from_file(mutable_empty_config, tmpdir): + config('--scope', 'site', 'add', 'config:template_dirs:test1') + contents = """spack: + config:: + template_dirs: [test2] +""" + + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + config('add', '-f', file) + output = config('get', 'config') + + assert output == """config: + template_dirs: [test2] +""" + + +def test_config_add_override_leaf_from_file(mutable_empty_config, tmpdir): + config('--scope', 'site', 'add', 'config:template_dirs:test1') + contents = """spack: + config: + template_dirs:: [test2] +""" + + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + config('add', '-f', file) + output = config('get', 'config') + + assert output == """config: + 'template_dirs:': [test2] +""" + + +def test_config_add_update_dict_from_file(mutable_empty_config, tmpdir): + config('add', 'packages:all:compiler:[gcc]') + + # contents to add to file + contents = """spack: + packages: + all: + version: + - 1.0.0 +""" + + # create temp file and add it to config + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + config('add', '-f', file) + + # get results + output = config('get', 'packages') + + expected = """packages: + all: + compiler: [gcc] + version: + - 1.0.0 +""" + + assert output == expected + + +def test_config_add_invalid_file_fails(tmpdir): + # contents to add to file + # invalid because version requires a list + contents = """spack: + packages: + all: + version: 1.0.0 +""" + + # create temp file and add it to config + file = str(tmpdir.join('spack.yaml')) + with open(file, 'w') as f: + f.write(contents) + + with pytest.raises( + (spack.config.ConfigFormatError) + ): + config('add', '-f', file) + + +def test_config_remove_value(mutable_empty_config): + config('add', 'config:dirty:true') + config('remove', 'config:dirty:true') + output = config('get', 'config') + + assert output == """config: {} +""" + + +def test_config_remove_alias_rm(mutable_empty_config): + config('add', 'config:dirty:true') + config('rm', 'config:dirty:true') + output = config('get', 'config') + + assert output == """config: {} +""" + + +def test_config_remove_dict(mutable_empty_config): + config('add', 'config:dirty:true') + config('rm', 'config:dirty') + output = config('get', 'config') + + assert output == """config: {} +""" + + +def test_remove_from_list(mutable_empty_config): + config('add', 'config:template_dirs:test1') + config('add', 'config:template_dirs:[test2]') + config('add', 'config:template_dirs:test3') + config('remove', 'config:template_dirs:test2') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test3 + - test1 +""" + + +def test_remove_list(mutable_empty_config): + config('add', 'config:template_dirs:test1') + config('add', 'config:template_dirs:[test2]') + config('add', 'config:template_dirs:test3') + config('remove', 'config:template_dirs:[test2]') + output = config('get', 'config') + + assert output == """config: + template_dirs: + - test3 + - test1 +""" + + +def test_config_add_to_env(mutable_empty_config, mutable_mock_env_path): + env = ev.create('test') + with env: + config('add', 'config:dirty:true') + output = config('get') + + expected = ev.default_manifest_yaml + expected += """ config: + dirty: true + +""" + assert output == expected + + +def test_config_remove_from_env(mutable_empty_config, mutable_mock_env_path): + env('create', 'test') + + with ev.read('test'): + config('add', 'config:dirty:true') + + with ev.read('test'): + config('rm', 'config:dirty') + output = config('get') + + expected = ev.default_manifest_yaml + expected += """ config: {} + +""" + assert output == expected diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 8212db6c214..88132e86727 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -576,7 +576,7 @@ def get_config_error(filename, schema, yaml_string): # parse and return error, or fail. try: - spack.config._read_config_file(filename, schema) + spack.config.read_config_file(filename, schema) except spack.config.ConfigFormatError as e: return e else: diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 8bd8deadce3..1efdd6ba959 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -374,6 +374,19 @@ def linux_os(): return LinuxOS(name=name, version=version) +@pytest.fixture(scope='session') +def default_config(): + """Isolates the default configuration from the user configs. + + This ensures we can test the real default configuration without having + tests fail when the user overrides the defaults that we test against.""" + defaults_path = os.path.join(spack.paths.etc_path, 'spack', 'defaults') + defaults_scope = spack.config.ConfigScope('defaults', defaults_path) + defaults_config = spack.config.Configuration(defaults_scope) + with use_configuration(defaults_config): + yield defaults_config + + @pytest.fixture(scope='session') def configuration_dir(tmpdir_factory, linux_os): """Copies mock configuration files in a temporary directory. Returns the @@ -436,6 +449,19 @@ def mutable_config(tmpdir_factory, configuration_dir): yield cfg +@pytest.fixture(scope='function') +def mutable_empty_config(tmpdir_factory, configuration_dir): + """Empty configuration that can be modified by the tests.""" + mutable_dir = tmpdir_factory.mktemp('mutable_config').join('tmp') + + cfg = spack.config.Configuration( + *[spack.config.ConfigScope(name, str(mutable_dir.join(name))) + for name in ['site', 'system', 'user']]) + + with use_configuration(cfg): + yield cfg + + @pytest.fixture() def mock_low_high_config(tmpdir): """Mocks two configuration scopes: 'low' and 'high'.""" diff --git a/lib/spack/spack/test/container/cli.py b/lib/spack/spack/test/container/cli.py index 8e5403f072f..f9b3b43f83b 100644 --- a/lib/spack/spack/test/container/cli.py +++ b/lib/spack/spack/test/container/cli.py @@ -9,8 +9,8 @@ containerize = spack.main.SpackCommand('containerize') -def test_command(configuration_dir, capsys): +def test_command(default_config, container_config_dir, capsys): with capsys.disabled(): - with fs.working_dir(configuration_dir): + with fs.working_dir(container_config_dir): output = containerize() assert 'FROM spack/ubuntu-bionic' in output diff --git a/lib/spack/spack/test/container/conftest.py b/lib/spack/spack/test/container/conftest.py index 802b34c5f89..65a89b07488 100644 --- a/lib/spack/spack/test/container/conftest.py +++ b/lib/spack/spack/test/container/conftest.py @@ -39,5 +39,5 @@ def dumper(configuration): @pytest.fixture() -def configuration_dir(minimal_configuration, config_dumper): +def container_config_dir(minimal_configuration, config_dumper): return config_dumper(minimal_configuration) diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py index fbdc085828e..f75ae7de2a3 100644 --- a/lib/spack/spack/test/container/docker.py +++ b/lib/spack/spack/test/container/docker.py @@ -42,7 +42,7 @@ def test_packages(minimal_configuration): assert p.list == pkgs -def test_ensure_render_works(minimal_configuration): +def test_ensure_render_works(minimal_configuration, default_config): # Here we just want to ensure that nothing is raised writer = writers.create(minimal_configuration) writer() diff --git a/lib/spack/spack/test/container/singularity.py b/lib/spack/spack/test/container/singularity.py index 445a119f6cb..ab342cacec5 100644 --- a/lib/spack/spack/test/container/singularity.py +++ b/lib/spack/spack/test/container/singularity.py @@ -13,7 +13,7 @@ def singularity_configuration(minimal_configuration): return minimal_configuration -def test_ensure_render_works(singularity_configuration): +def test_ensure_render_works(default_config, singularity_configuration): container_config = singularity_configuration['spack']['container'] assert container_config['format'] == 'singularity' # Here we just want to ensure that nothing is raised diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index e2db47ba301..ffd75964164 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -579,7 +579,7 @@ _spack_config() { then SPACK_COMPREPLY="-h --help --scope" else - SPACK_COMPREPLY="get blame edit list" + SPACK_COMPREPLY="get blame edit list add remove rm" fi } @@ -614,6 +614,33 @@ _spack_config_list() { SPACK_COMPREPLY="-h --help" } +_spack_config_add() { + if $list_options + then + SPACK_COMPREPLY="-h --help -f --file" + else + SPACK_COMPREPLY="" + fi +} + +_spack_config_remove() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="" + fi +} + +_spack_config_rm() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="" + fi +} + _spack_configure() { if $list_options then From 5c712a03a1262b97fc920c86a2125fef7c806762 Mon Sep 17 00:00:00 2001 From: manifest Date: Thu, 25 Jun 2020 18:13:33 +0930 Subject: [PATCH 22/38] msmc2: added package at v2.1.2 (#17170) Co-authored-by: Adam J. Stewart --- .../repos/builtin/packages/msmc2/package.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 var/spack/repos/builtin/packages/msmc2/package.py diff --git a/var/spack/repos/builtin/packages/msmc2/package.py b/var/spack/repos/builtin/packages/msmc2/package.py new file mode 100644 index 00000000000..efd0bec341c --- /dev/null +++ b/var/spack/repos/builtin/packages/msmc2/package.py @@ -0,0 +1,43 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Msmc2(MakefilePackage): + """ + This program implements MSMC2, a method to infer population size history + and population separation history from whole genome sequencing data. + """ + + homepage = "https://github.com/stschiff/msmc2" + url = "https://github.com/stschiff/msmc2/archive/v2.1.1.tar.gz" + + maintainers = ['robqiao'] + + version('2.1.2', sha256='c8c2b6ed6ef1b5d6e6e15ec0d968288e5bdae2bea1f3b4ec790f599e96bb26cd') + version('2.1.1', sha256='151dd75a8b0371ff94eed9504a7a73a2924e10466de30c230c1bb3c35a1a0913') + version('2.0.2', sha256='91152b2494342148ff84a1e5c6d2f5f0d53acba12722cd68ff5807ba4e82af55') + version('2.0.1', sha256='97e859e6f08689baf29d3c61b6904cfa5a292f8ce7b3532e055ce3047d8472f4') + version('2.0.0', sha256='9de38239f6e729a0f6f492ca671e2e70541eb5db558d816e64184c06611a1c7e') + + depends_on('gsl', type=('build', 'run')) + depends_on('dmd@:2.081.0', type='build') + + def edit(self, spec, prefix): + # Set DMD compiler + filter_file('dmd', + join_path(self.spec['dmd'].prefix.linux.bin64, 'dmd'), + 'Makefile', string=True) + + gsllibdir = spec['gsl'].libs.directories[0] + + # Set GSLDIR + filter_file('GSLDIR=/usr/local/lib', + 'GSLDIR={0}'.format(gsllibdir), + 'Makefile', string=True) + + def install(self, spec, prefix): + install_tree('build/release', prefix.bin) From a31c115d799c364f60e82fed02ba1fcc80bd935f Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 25 Jun 2020 06:27:27 -0700 Subject: [PATCH 23/38] Fix global activation check for upstream extendees (#17231) * short-circuit is_activated check when the extendee is installed upstream * add test for checking activation status of packages with an extendee installed upstream --- lib/spack/spack/package.py | 5 ++++ lib/spack/spack/test/test_activations.py | 29 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dc32effbec9..5c4ff23d406 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1027,6 +1027,11 @@ def is_activated(self, view): if not self.is_extension: raise ValueError( "is_activated called on package that is not an extension.") + if self.extendee_spec.package.installed_upstream: + # If this extends an upstream package, it cannot be activated for + # it. This bypasses construction of the extension map, which can + # can fail when run in the context of a downstream Spack instance + return False extensions_layout = view.extensions_layout exts = extensions_layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) diff --git a/lib/spack/spack/test/test_activations.py b/lib/spack/spack/test/test_activations.py index eaac98f3189..d1780b89639 100644 --- a/lib/spack/spack/test/test_activations.py +++ b/lib/spack/spack/test/test_activations.py @@ -408,3 +408,32 @@ def test_perl_activation_view(tmpdir, perl_and_extension_dirs, assert not os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool')) assert os.path.exists(os.path.join(view_dir, 'bin/perl-ext-tool')) + + +def test_is_activated_upstream_extendee(tmpdir, builtin_and_mock_packages, + monkeypatch): + """When an extendee is installed upstream, make sure that the extension + spec is never considered to be globally activated for it. + """ + extendee_spec = spack.spec.Spec('python') + extendee_spec._concrete = True + + python_name = 'python' + tmpdir.ensure(python_name, dir=True) + + python_prefix = str(tmpdir.join(python_name)) + # Set the prefix on the package's spec reference because that is a copy of + # the original spec + extendee_spec.package.spec.prefix = python_prefix + monkeypatch.setattr(extendee_spec.package.__class__, + 'installed_upstream', True) + + ext_name = 'py-extension1' + tmpdir.ensure(ext_name, dir=True) + ext_pkg = create_ext_pkg( + ext_name, str(tmpdir.join(ext_name)), extendee_spec, monkeypatch) + + # The view should not be checked at all if the extendee is installed + # upstream, so use 'None' here + mock_view = None + assert not ext_pkg.is_activated(mock_view) From 14599f09be6cd6eac4dc4d9d2d01a0ad001ae3f5 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 25 Jun 2020 18:18:48 +0200 Subject: [PATCH 24/38] Separate Apple Clang from LLVM Clang (#17110) * Separate Apple Clang from LLVM Clang Apple Clang is a compiler of its own. All places referring to "-apple" suffix have been updated. * Hack to use a dash in 'apple-clang' To be able to use autodoc from Sphinx we need a valid Python name for the module that contains Apple's Clang code. * Updated packages to account for the existence of apple-clang Co-authored-by: Adam J. Stewart * Added unit test for XCode related functions Co-authored-by: Gregory Becker Co-authored-by: Adam J. Stewart --- etc/spack/defaults/darwin/packages.yaml | 6 +- lib/spack/docs/basic_usage.rst | 4 +- lib/spack/docs/getting_started.rst | 2 +- lib/spack/docs/packaging_guide.rst | 8 +- lib/spack/docs/repositories.rst | 20 +- lib/spack/llnl/util/cpu/microarchitecture.py | 29 ++- .../llnl/util/cpu/microarchitectures.json | 8 +- lib/spack/spack/ci.py | 12 +- lib/spack/spack/compilers/__init__.py | 16 +- lib/spack/spack/compilers/apple_clang.py | 165 ++++++++++++++ lib/spack/spack/compilers/clang.py | 209 +++--------------- lib/spack/spack/spec.py | 8 +- lib/spack/spack/test/architecture.py | 6 +- lib/spack/spack/test/compilers/basics.py | 140 ++++++++++-- lib/spack/spack/test/compilers/detection.py | 17 +- lib/spack/spack/test/concretize.py | 2 +- lib/spack/spack/test/conftest.py | 2 +- .../spack/test/data/config/compilers.yaml | 2 +- .../builtin/packages/allpaths-lg/package.py | 1 + .../builtin/packages/aluminum/package.py | 4 +- .../repos/builtin/packages/amber/package.py | 10 +- .../repos/builtin/packages/amrex/package.py | 1 + .../repos/builtin/packages/ape/package.py | 8 +- .../repos/builtin/packages/atlas/package.py | 2 +- .../builtin/packages/binutils/package.py | 7 +- .../repos/builtin/packages/boost/package.py | 4 +- .../repos/builtin/packages/bucky/package.py | 1 + .../repos/builtin/packages/cbench/package.py | 1 + .../repos/builtin/packages/cp2k/package.py | 1 + .../repos/builtin/packages/dealii/package.py | 2 +- .../builtin/packages/elfutils/package.py | 1 + .../repos/builtin/packages/f18/package.py | 3 + .../repos/builtin/packages/fftw/package.py | 6 +- .../builtin/packages/fujitsu-mpi/package.py | 1 + .../builtin/packages/gitconddb/package.py | 1 + .../builtin/packages/graphviz/package.py | 7 + .../repos/builtin/packages/hpgmg/package.py | 4 +- .../builtin/packages/hydrogen/package.py | 4 +- .../builtin/packages/intel-tbb/package.py | 4 +- .../repos/builtin/packages/kahip/package.py | 1 + .../builtin/packages/kentutils/package.py | 1 + .../repos/builtin/packages/lbann/package.py | 12 +- .../repos/builtin/packages/libceed/package.py | 12 +- .../repos/builtin/packages/libcerf/package.py | 2 +- .../repos/builtin/packages/librsb/package.py | 1 + .../repos/builtin/packages/likwid/package.py | 10 +- .../builtin/packages/maverick/package.py | 1 + .../repos/builtin/packages/mpich/package.py | 1 + .../builtin/packages/mpilander/package.py | 1 + .../repos/builtin/packages/musl/package.py | 4 +- .../repos/builtin/packages/octopus/package.py | 6 +- .../repos/builtin/packages/onednn/package.py | 4 +- .../builtin/packages/openblas/package.py | 17 +- .../repos/builtin/packages/pdt/package.py | 3 +- .../repos/builtin/packages/petsc/package.py | 8 +- .../repos/builtin/packages/pixman/package.py | 3 +- .../repos/builtin/packages/plasma/package.py | 1 + .../repos/builtin/packages/precice/package.py | 1 + .../repos/builtin/packages/py-dgl/package.py | 4 +- .../packages/py-exodus-bundler/package.py | 1 + .../repos/builtin/packages/py-flye/package.py | 2 +- .../packages/py-scikit-learn/package.py | 6 +- .../builtin/packages/py-torch/package.py | 2 +- .../repos/builtin/packages/qmcpack/package.py | 1 + .../repos/builtin/packages/qt/package.py | 2 +- .../repos/builtin/packages/raxml/package.py | 1 + .../repos/builtin/packages/sgpp/package.py | 1 + .../repos/builtin/packages/silo/package.py | 7 + .../builtin/packages/snap-berkeley/package.py | 1 + .../builtin/packages/sqlitebrowser/package.py | 1 + .../builtin/packages/suite-sparse/package.py | 2 +- .../builtin/packages/sundials/package.py | 2 +- .../builtin/packages/trilinos/package.py | 4 +- .../repos/builtin/packages/unblur/package.py | 1 + .../builtin/packages/valgrind/package.py | 2 +- .../repos/builtin/packages/vtk/package.py | 4 +- .../repos/builtin/packages/warpx/package.py | 3 +- .../repos/builtin/packages/xios/package.py | 13 +- .../builtin/packages/xsdktrilinos/package.py | 4 +- .../builtin/packages/yaml-cpp/package.py | 4 +- 80 files changed, 536 insertions(+), 350 deletions(-) create mode 100644 lib/spack/spack/compilers/apple_clang.py diff --git a/etc/spack/defaults/darwin/packages.yaml b/etc/spack/defaults/darwin/packages.yaml index 1150f6bab72..b2bcd560c67 100644 --- a/etc/spack/defaults/darwin/packages.yaml +++ b/etc/spack/defaults/darwin/packages.yaml @@ -15,7 +15,11 @@ # ------------------------------------------------------------------------- packages: all: - compiler: [clang, gcc, intel] + compiler: + - apple-clang + - clang + - gcc + - intel providers: elf: [libelf] unwind: [apple-libunwind] diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 8acebeb0e66..93c5858d938 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -627,8 +627,8 @@ output metadata on specs and all dependencies as json: "target": "x86_64" }, "compiler": { - "name": "clang", - "version": "10.0.0-apple" + "name": "apple-clang", + "version": "10.0.0" }, "namespace": "builtin", "parameters": { diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 1fbddbd4a98..226b1f0883c 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -478,7 +478,7 @@ Fortran. cxx: /usr/bin/clang++ f77: /path/to/bin/gfortran fc: /path/to/bin/gfortran - spec: clang@11.0.0-apple + spec: apple-clang@11.0.0 If you used Spack to install GCC, you can get the installation prefix by diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 533036852b3..d3a888b1fcd 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1675,15 +1675,15 @@ can see the patches that would be applied to ``m4``:: Concretized -------------------------------- - m4@1.4.18%clang@9.0.0-apple patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64 - ^libsigsegv@2.11%clang@9.0.0-apple arch=darwin-highsierra-x86_64 + m4@1.4.18%apple-clang@9.0.0 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64 + ^libsigsegv@2.11%apple-clang@9.0.0 arch=darwin-highsierra-x86_64 You can also see patches that have been applied to installed packages with ``spack find -v``:: $ spack find -v m4 ==> 1 installed package - -- darwin-highsierra-x86_64 / clang@9.0.0-apple ----------------- + -- darwin-highsierra-x86_64 / apple-clang@9.0.0 ----------------- m4@1.4.18 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv .. _cmd-spack-resource: @@ -1713,7 +1713,7 @@ wonder where the extra boost patches are coming from:: $ spack spec dealii ^boost@1.68.0 ^hdf5+fortran | grep '\^boost' ^boost@1.68.0 - ^boost@1.68.0%clang@9.0.0-apple+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64 + ^boost@1.68.0%apple-clang@9.0.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64 $ spack resource show b37164268 b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f path: /home/spackuser/src/spack/var/spack/repos/builtin/packages/dealii/boost_1.68.0.patch diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 6c2050a4be7..4a2c1638867 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -280,16 +280,16 @@ you install it, you can use ``spack spec -N``: Concretized -------------------------------- - builtin.hdf5@1.10.0-patch1%clang@7.0.2-apple+cxx~debug+fortran+mpi+shared~szip~threadsafe arch=darwin-elcapitan-x86_64 - ^builtin.openmpi@2.0.1%clang@7.0.2-apple~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt arch=darwin-elcapitan-x86_64 - ^builtin.hwloc@1.11.4%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.libpciaccess@0.13.4%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.libtool@2.4.6%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.m4@1.4.17%clang@7.0.2-apple+sigsegv arch=darwin-elcapitan-x86_64 - ^builtin.libsigsegv@2.10%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.pkg-config@0.29.1%clang@7.0.2-apple+internal_glib arch=darwin-elcapitan-x86_64 - ^builtin.util-macros@1.19.0%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.zlib@1.2.8%clang@7.0.2-apple+pic arch=darwin-elcapitan-x86_64 + builtin.hdf5@1.10.0-patch1%apple-clang@7.0.2+cxx~debug+fortran+mpi+shared~szip~threadsafe arch=darwin-elcapitan-x86_64 + ^builtin.openmpi@2.0.1%apple-clang@7.0.2~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt arch=darwin-elcapitan-x86_64 + ^builtin.hwloc@1.11.4%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.libpciaccess@0.13.4%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.libtool@2.4.6%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.m4@1.4.17%apple-clang@7.0.2+sigsegv arch=darwin-elcapitan-x86_64 + ^builtin.libsigsegv@2.10%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.pkg-config@0.29.1%apple-clang@7.0.2+internal_glib arch=darwin-elcapitan-x86_64 + ^builtin.util-macros@1.19.0%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.zlib@1.2.8%apple-clang@7.0.2+pic arch=darwin-elcapitan-x86_64 .. warning:: diff --git a/lib/spack/llnl/util/cpu/microarchitecture.py b/lib/spack/llnl/util/cpu/microarchitecture.py index f507837f890..284bfebb7e3 100644 --- a/lib/spack/llnl/util/cpu/microarchitecture.py +++ b/lib/spack/llnl/util/cpu/microarchitecture.py @@ -204,10 +204,22 @@ def optimization_flags(self, compiler, version): compiler (str): name of the compiler to be used version (str): version of the compiler to be used """ - # If we don't have information on compiler return an empty string - if compiler not in self.compilers: + # If we don't have information on compiler at all + # return an empty string + if compiler not in self.family.compilers: return '' + # If we have information but it stops before this + # microarchitecture, fall back to the best known target + if compiler not in self.compilers: + best_target = [ + x for x in self.ancestors if compiler in x.compilers + ][0] + msg = ("'{0}' compiler is known to optimize up to the '{1}'" + " microarchitecture in the '{2}' architecture family") + msg = msg.format(compiler, best_target, best_target.family) + raise UnsupportedMicroarchitecture(msg) + # If we have information on this compiler we need to check the # version being used compiler_info = self.compilers[compiler] @@ -219,15 +231,10 @@ def optimization_flags(self, compiler, version): def satisfies_constraint(entry, version): min_version, max_version = entry['versions'].split(':') - # Check version suffixes - min_version, min_suffix = version_components(min_version) - max_version, max_suffix = version_components(max_version) - version, suffix = version_components(version) - - # If the suffixes are not all equal there's no match - if ((suffix != min_suffix and min_version) or - (suffix != max_suffix and max_version)): - return False + # Extract numeric part of the version + min_version, _ = version_components(min_version) + max_version, _ = version_components(max_version) + version, _ = version_components(version) # Assume compiler versions fit into semver tuplify = lambda x: tuple(int(y) for y in x.split('.')) diff --git a/lib/spack/llnl/util/cpu/microarchitectures.json b/lib/spack/llnl/util/cpu/microarchitectures.json index 51411d4a24c..e94146bf852 100644 --- a/lib/spack/llnl/util/cpu/microarchitectures.json +++ b/lib/spack/llnl/util/cpu/microarchitectures.json @@ -61,12 +61,14 @@ "flags": "-march={name} -mtune={name}" } ], - "clang": [ + "apple-clang": [ { - "versions": "0.0.0-apple:", + "versions": ":", "name": "x86-64", "flags": "-march={name}" - }, + } + ], + "clang": [ { "versions": ":", "name": "x86-64", diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index ce74abf29cc..230f25ccee2 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -345,18 +345,18 @@ def compute_spec_deps(spec_list): ], "specs": [ { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "readline@7.0%clang@9.1.0-apple arch=darwin-highs...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-highs...", "label": "readline/ip6aiun" }, { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "ncurses@6.1%clang@9.1.0-apple arch=darwin-highsi...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "ncurses@6.1%apple-clang@9.1.0 arch=darwin-highsi...", "label": "ncurses/y43rifz" }, { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "pkgconf@1.5.4%clang@9.1.0-apple arch=darwin-high...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "pkgconf@1.5.4%apple-clang@9.1.0 arch=darwin-high...", "label": "pkgconf/eg355zb" } ] diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 771459f7f3c..dfa750cc4d4 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -245,7 +245,9 @@ def supported_compilers(): See available_compilers() to get a list of all the available versions of supported compilers. """ - return sorted(name for name in + # Hack to be able to call the compiler `apple-clang` while still + # using a valid python name for the module + return sorted(name if name != 'apple_clang' else 'apple-clang' for name in llnl.util.lang.list_modules(spack.paths.compilers_path)) @@ -469,7 +471,13 @@ def class_for_compiler_name(compiler_name): """Given a compiler module name, get the corresponding Compiler class.""" assert(supported(compiler_name)) - file_path = os.path.join(spack.paths.compilers_path, compiler_name + ".py") + # Hack to be able to call the compiler `apple-clang` while still + # using a valid python name for the module + module_name = compiler_name + if compiler_name == 'apple-clang': + module_name = compiler_name.replace('-', '_') + + file_path = os.path.join(spack.paths.compilers_path, module_name + ".py") compiler_mod = simp.load_source(_imported_compilers_module, file_path) cls = getattr(compiler_mod, mod_to_class(compiler_name)) @@ -662,7 +670,7 @@ def _default(cmp_id, paths): operating_system, compiler_name, version = cmp_id compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, version) - paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')] + paths = [paths.get(x, None) for x in ('cc', 'cxx', 'f77', 'fc')] target = cpu.host() compiler = compiler_cls( spec, operating_system, str(target.family), paths @@ -716,6 +724,8 @@ def name_matches(name, name_list): toolchains.add(compiler_cls.__name__) if len(toolchains) > 1: + if toolchains == set(['Clang', 'AppleClang']): + return False tty.debug("[TOOLCHAINS] {0}".format(toolchains)) return True diff --git a/lib/spack/spack/compilers/apple_clang.py b/lib/spack/spack/compilers/apple_clang.py new file mode 100644 index 00000000000..8ef58550ea9 --- /dev/null +++ b/lib/spack/spack/compilers/apple_clang.py @@ -0,0 +1,165 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path +import re +import shutil + +import llnl.util.tty as tty +import llnl.util.lang +import spack.compiler +import spack.compilers.clang +import spack.util.executable +import spack.version + + +class AppleClang(spack.compilers.clang.Clang): + openmp_flag = "-Xpreprocessor -fopenmp" + + @classmethod + @llnl.util.lang.memoized + def extract_version_from_output(cls, output): + ver = 'unknown' + match = re.search( + # Apple's LLVM compiler has its own versions, so suffix them. + r'^Apple (?:LLVM|clang) version ([^ )]+)', output + ) + if match: + ver = match.group(match.lastindex) + return ver + + @property + def cxx11_flag(self): + # Adapted from CMake's AppleClang-CXX rules + # Spack's AppleClang detection only valid from Xcode >= 4.6 + if self.version < spack.version.ver('4.0.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++11 standard", "cxx11_flag", "Xcode < 4.0.0" + ) + return "-std=c++11" + + @property + def cxx14_flag(self): + # Adapted from CMake's rules for AppleClang + if self.version < spack.version.ver('5.1.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++14 standard", "cxx14_flag", "Xcode < 5.1.0" + ) + elif self.version < spack.version.ver('6.1.0'): + return "-std=c++1y" + + return "-std=c++14" + + @property + def cxx17_flag(self): + # Adapted from CMake's rules for AppleClang + if self.version < spack.version.ver('6.1.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++17 standard", "cxx17_flag", "Xcode < 6.1.0" + ) + return "-std=c++1z" + + def setup_custom_environment(self, pkg, env): + """Set the DEVELOPER_DIR environment for the Xcode toolchain. + + On macOS, not all buildsystems support querying CC and CXX for the + compilers to use and instead query the Xcode toolchain for what + compiler to run. This side-steps the spack wrappers. In order to inject + spack into this setup, we need to copy (a subset of) Xcode.app and + replace the compiler executables with symlinks to the spack wrapper. + Currently, the stage is used to store the Xcode.app copies. We then set + the 'DEVELOPER_DIR' environment variables to cause the xcrun and + related tools to use this Xcode.app. + """ + super(AppleClang, self).setup_custom_environment(pkg, env) + + if not pkg.use_xcode: + # if we do it for all packages, we get into big troubles with MPI: + # filter_compilers(self) will use mockup XCode compilers on macOS + # with Clang. Those point to Spack's compiler wrappers and + # consequently render MPI non-functional outside of Spack. + return + + # Use special XCode versions of compiler wrappers when using XCode + # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX + xcrun = spack.util.executable.Executable('xcrun') + xcode_clang = xcrun('-f', 'clang', output=str).strip() + xcode_clangpp = xcrun('-f', 'clang++', output=str).strip() + env.set('SPACK_CC', xcode_clang, force=True) + env.set('SPACK_CXX', xcode_clangpp, force=True) + + xcode_select = spack.util.executable.Executable('xcode-select') + + # Get the path of the active developer directory + real_root = xcode_select('--print-path', output=str).strip() + + # The path name can be used to determine whether the full Xcode suite + # or just the command-line tools are installed + if real_root.endswith('Developer'): + # The full Xcode suite is installed + pass + else: + if real_root.endswith('CommandLineTools'): + # Only the command-line tools are installed + msg = 'It appears that you have the Xcode command-line tools ' + msg += 'but not the full Xcode suite installed.\n' + + else: + # Xcode is not installed + msg = 'It appears that you do not have Xcode installed.\n' + + msg += 'In order to use Spack to build the requested application, ' + msg += 'you need the full Xcode suite. It can be installed ' + msg += 'through the App Store. Make sure you launch the ' + msg += 'application and accept the license agreement.\n' + + raise OSError(msg) + + real_root = os.path.dirname(os.path.dirname(real_root)) + developer_root = os.path.join(spack.stage.get_stage_root(), + 'xcode-select', + self.name, + str(self.version)) + xcode_link = os.path.join(developer_root, 'Xcode.app') + + if not os.path.exists(developer_root): + tty.warn('Copying Xcode from %s to %s in order to add spack ' + 'wrappers to it. Please do not interrupt.' + % (real_root, developer_root)) + + # We need to make a new Xcode.app instance, but with symlinks to + # the spack wrappers for the compilers it ships. This is necessary + # because some projects insist on just asking xcrun and related + # tools where the compiler runs. These tools are very hard to trick + # as they do realpath and end up ignoring the symlinks in a + # "softer" tree of nothing but symlinks in the right places. + shutil.copytree( + real_root, developer_root, symlinks=True, + ignore=shutil.ignore_patterns( + 'AppleTV*.platform', 'Watch*.platform', 'iPhone*.platform', + 'Documentation', 'swift*' + )) + + real_dirs = [ + 'Toolchains/XcodeDefault.xctoolchain/usr/bin', + 'usr/bin', + ] + + bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] + + for real_dir in real_dirs: + dev_dir = os.path.join(developer_root, + 'Contents', + 'Developer', + real_dir) + for fname in os.listdir(dev_dir): + if fname in bins: + os.unlink(os.path.join(dev_dir, fname)) + os.symlink( + os.path.join(spack.paths.build_env_path, 'cc'), + os.path.join(dev_dir, fname)) + + os.symlink(developer_root, xcode_link) + + env.set('DEVELOPER_DIR', xcode_link) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 846c3609dc8..f158ff32762 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -4,17 +4,11 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re -import os import sys -from shutil import copytree, ignore_patterns import llnl.util.lang -import llnl.util.tty as tty -import spack.paths -import spack.stage from spack.compiler import Compiler, UnsupportedCompilerFlag -from spack.util.executable import Executable from spack.version import ver @@ -88,88 +82,41 @@ def link_paths(self): return link_paths - @property - def is_apple(self): - ver_string = str(self.version) - return ver_string.endswith('-apple') - @property def verbose_flag(self): return "-v" - @property - def openmp_flag(self): - if self.is_apple: - return "-Xpreprocessor -fopenmp" - else: - return "-fopenmp" + openmp_flag = "-fopenmp" @property def cxx11_flag(self): - if self.is_apple: - # Adapted from CMake's AppleClang-CXX rules - # Spack's AppleClang detection only valid from Xcode >= 4.6 - if self.version < ver('4.0.0'): - raise UnsupportedCompilerFlag(self, - "the C++11 standard", - "cxx11_flag", - "Xcode < 4.0.0") - else: - return "-std=c++11" - else: - if self.version < ver('3.3'): - raise UnsupportedCompilerFlag(self, - "the C++11 standard", - "cxx11_flag", - "< 3.3") - else: - return "-std=c++11" + if self.version < ver('3.3'): + raise UnsupportedCompilerFlag( + self, "the C++11 standard", "cxx11_flag", "< 3.3" + ) + return "-std=c++11" @property def cxx14_flag(self): - if self.is_apple: - # Adapted from CMake's rules for AppleClang - if self.version < ver('5.1.0'): - raise UnsupportedCompilerFlag(self, - "the C++14 standard", - "cxx14_flag", - "Xcode < 5.1.0") - elif self.version < ver('6.1.0'): - return "-std=c++1y" - else: - return "-std=c++14" - else: - if self.version < ver('3.4'): - raise UnsupportedCompilerFlag(self, - "the C++14 standard", - "cxx14_flag", - "< 3.5") - elif self.version < ver('3.5'): - return "-std=c++1y" - else: - return "-std=c++14" + if self.version < ver('3.4'): + raise UnsupportedCompilerFlag( + self, "the C++14 standard", "cxx14_flag", "< 3.5" + ) + elif self.version < ver('3.5'): + return "-std=c++1y" + + return "-std=c++14" @property def cxx17_flag(self): - if self.is_apple: - # Adapted from CMake's rules for AppleClang - if self.version < ver('6.1.0'): - raise UnsupportedCompilerFlag(self, - "the C++17 standard", - "cxx17_flag", - "Xcode < 6.1.0") - else: - return "-std=c++1z" - else: - if self.version < ver('3.5'): - raise UnsupportedCompilerFlag(self, - "the C++17 standard", - "cxx17_flag", - "< 3.5") - elif self.version < ver('5.0'): - return "-std=c++1z" - else: - return "-std=c++17" + if self.version < ver('3.5'): + raise UnsupportedCompilerFlag( + self, "the C++17 standard", "cxx17_flag", "< 3.5" + ) + elif self.version < ver('5.0'): + return "-std=c++1z" + + return "-std=c++17" @property def c99_flag(self): @@ -207,9 +154,10 @@ def fc_pic_flag(self): @llnl.util.lang.memoized def extract_version_from_output(cls, output): ver = 'unknown' + if 'Apple' in output: + return ver + match = re.search( - # Apple's LLVM compiler has its own versions, so suffix them. - r'^Apple (?:LLVM|clang) version ([^ )]+)|' # Normal clang compiler versions are left as-is r'clang version ([^ )]+)-svn[~.\w\d-]*|' # Don't include hyphenated patch numbers in the version @@ -219,8 +167,7 @@ def extract_version_from_output(cls, output): output ) if match: - suffix = '-apple' if match.lastindex == 1 else '' - ver = match.group(match.lastindex) + suffix + ver = match.group(match.lastindex) return ver @classmethod @@ -235,107 +182,3 @@ def fc_version(cls, fc): @classmethod def f77_version(cls, f77): return cls.fc_version(f77) - - def setup_custom_environment(self, pkg, env): - """Set the DEVELOPER_DIR environment for the Xcode toolchain. - - On macOS, not all buildsystems support querying CC and CXX for the - compilers to use and instead query the Xcode toolchain for what - compiler to run. This side-steps the spack wrappers. In order to inject - spack into this setup, we need to copy (a subset of) Xcode.app and - replace the compiler executables with symlinks to the spack wrapper. - Currently, the stage is used to store the Xcode.app copies. We then set - the 'DEVELOPER_DIR' environment variables to cause the xcrun and - related tools to use this Xcode.app. - """ - super(Clang, self).setup_custom_environment(pkg, env) - - if not self.is_apple or not pkg.use_xcode: - # if we do it for all packages, we get into big troubles with MPI: - # filter_compilers(self) will use mockup XCode compilers on macOS - # with Clang. Those point to Spack's compiler wrappers and - # consequently render MPI non-functional outside of Spack. - return - - # Use special XCode versions of compiler wrappers when using XCode - # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX - xcrun = Executable('xcrun') - xcode_clang = xcrun('-f', 'clang', output=str).strip() - xcode_clangpp = xcrun('-f', 'clang++', output=str).strip() - env.set('SPACK_CC', xcode_clang, force=True) - env.set('SPACK_CXX', xcode_clangpp, force=True) - - xcode_select = Executable('xcode-select') - - # Get the path of the active developer directory - real_root = xcode_select('--print-path', output=str).strip() - - # The path name can be used to determine whether the full Xcode suite - # or just the command-line tools are installed - if real_root.endswith('Developer'): - # The full Xcode suite is installed - pass - else: - if real_root.endswith('CommandLineTools'): - # Only the command-line tools are installed - msg = 'It appears that you have the Xcode command-line tools ' - msg += 'but not the full Xcode suite installed.\n' - - else: - # Xcode is not installed - msg = 'It appears that you do not have Xcode installed.\n' - - msg += 'In order to use Spack to build the requested application, ' - msg += 'you need the full Xcode suite. It can be installed ' - msg += 'through the App Store. Make sure you launch the ' - msg += 'application and accept the license agreement.\n' - - raise OSError(msg) - - real_root = os.path.dirname(os.path.dirname(real_root)) - developer_root = os.path.join(spack.stage.get_stage_root(), - 'xcode-select', - self.name, - str(self.version)) - xcode_link = os.path.join(developer_root, 'Xcode.app') - - if not os.path.exists(developer_root): - tty.warn('Copying Xcode from %s to %s in order to add spack ' - 'wrappers to it. Please do not interrupt.' - % (real_root, developer_root)) - - # We need to make a new Xcode.app instance, but with symlinks to - # the spack wrappers for the compilers it ships. This is necessary - # because some projects insist on just asking xcrun and related - # tools where the compiler runs. These tools are very hard to trick - # as they do realpath and end up ignoring the symlinks in a - # "softer" tree of nothing but symlinks in the right places. - copytree(real_root, developer_root, symlinks=True, - ignore=ignore_patterns('AppleTV*.platform', - 'Watch*.platform', - 'iPhone*.platform', - 'Documentation', - 'swift*')) - - real_dirs = [ - 'Toolchains/XcodeDefault.xctoolchain/usr/bin', - 'usr/bin', - ] - - bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] - - for real_dir in real_dirs: - dev_dir = os.path.join(developer_root, - 'Contents', - 'Developer', - real_dir) - for fname in os.listdir(dev_dir): - if fname in bins: - os.unlink(os.path.join(dev_dir, fname)) - os.symlink( - os.path.join(spack.paths.build_env_path, 'cc'), - os.path.join(dev_dir, fname)) - - os.symlink(developer_root, xcode_link) - - env.set('DEVELOPER_DIR', xcode_link) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b09d72003e6..b23142e101d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1448,8 +1448,8 @@ def to_node_dict(self, hash=ht.dag_hash): 'target': 'x86_64', }, 'compiler': { - 'name': 'clang', - 'version': '10.0.0-apple', + 'name': 'apple-clang', + 'version': '10.0.0', }, 'namespace': 'builtin', 'parameters': { @@ -1554,8 +1554,8 @@ def to_dict(self, hash=ht.dag_hash): 'target': 'x86_64', }, 'compiler': { - 'name': 'clang', - 'version': '10.0.0-apple', + 'name': 'apple-clang', + 'version': '10.0.0', }, 'namespace': 'builtin', 'parameters': { diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 48cec134d27..80d3fc72c06 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -176,8 +176,8 @@ def test_arch_spec_container_semantic(item, architecture_str): # Check mixed toolchains ('clang@8.0.0', 'broadwell', ''), ('clang@3.5', 'x86_64', '-march=x86-64 -mtune=generic'), - # Check clang compilers with 'apple' suffix - ('clang@9.1.0-apple', 'x86_64', '-march=x86-64') + # Check Apple's Clang compilers + ('apple-clang@9.1.0', 'x86_64', '-march=x86-64') ]) @pytest.mark.filterwarnings("ignore:microarchitecture specific") def test_optimization_flags( @@ -200,7 +200,7 @@ def test_optimization_flags( '-march=icelake-client -mtune=icelake-client'), # Check that the special case for Apple's clang is treated correctly # i.e. it won't try to detect the version again - (spack.spec.CompilerSpec('clang@9.1.0-apple'), None, 'x86_64', + (spack.spec.CompilerSpec('apple-clang@9.1.0'), None, 'x86_64', '-march=x86-64'), ]) def test_optimization_flags_with_custom_versions( diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 32f298ae2b3..825975b21aa 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -7,6 +7,7 @@ import sys import os +import shutil from copy import copy from six import iteritems @@ -16,6 +17,8 @@ import spack.spec import spack.compiler import spack.compilers as compilers +import spack.spec +import spack.util.environment from spack.compiler import Compiler from spack.util.executable import ProcessError @@ -130,7 +133,7 @@ def test_compiler_flags_from_config_are_grouped(): # Utility function to test most flags. default_compiler_entry = { - 'spec': 'clang@2.0.0-apple', + 'spec': 'apple-clang@2.0.0', 'operating_system': 'foo-os', 'paths': { 'cc': 'cc-path', @@ -370,26 +373,27 @@ def test_cce_flags(): 'cce@1.0') -def test_clang_flags(): - # Apple Clang. +def test_apple_clang_flags(): supported_flag_test( - "openmp_flag", "-Xpreprocessor -fopenmp", "clang@2.0.0-apple") - unsupported_flag_test("cxx11_flag", "clang@2.0.0-apple") - supported_flag_test("cxx11_flag", "-std=c++11", "clang@4.0.0-apple") - unsupported_flag_test("cxx14_flag", "clang@5.0.0-apple") - supported_flag_test("cxx14_flag", "-std=c++1y", "clang@5.1.0-apple") - supported_flag_test("cxx14_flag", "-std=c++14", "clang@6.1.0-apple") - unsupported_flag_test("cxx17_flag", "clang@6.0.0-apple") - supported_flag_test("cxx17_flag", "-std=c++1z", "clang@6.1.0-apple") - supported_flag_test("c99_flag", "-std=c99", "clang@6.1.0-apple") - unsupported_flag_test("c11_flag", "clang@6.0.0-apple") - supported_flag_test("c11_flag", "-std=c11", "clang@6.1.0-apple") - supported_flag_test("cc_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("cxx_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("f77_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("fc_pic_flag", "-fPIC", "clang@2.0.0-apple") + "openmp_flag", "-Xpreprocessor -fopenmp", "apple-clang@2.0.0" + ) + unsupported_flag_test("cxx11_flag", "apple-clang@2.0.0") + supported_flag_test("cxx11_flag", "-std=c++11", "apple-clang@4.0.0") + unsupported_flag_test("cxx14_flag", "apple-clang@5.0.0") + supported_flag_test("cxx14_flag", "-std=c++1y", "apple-clang@5.1.0") + supported_flag_test("cxx14_flag", "-std=c++14", "apple-clang@6.1.0") + unsupported_flag_test("cxx17_flag", "apple-clang@6.0.0") + supported_flag_test("cxx17_flag", "-std=c++1z", "apple-clang@6.1.0") + supported_flag_test("c99_flag", "-std=c99", "apple-clang@6.1.0") + unsupported_flag_test("c11_flag", "apple-clang@6.0.0") + supported_flag_test("c11_flag", "-std=c11", "apple-clang@6.1.0") + supported_flag_test("cc_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("cxx_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("f77_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("fc_pic_flag", "-fPIC", "apple-clang@2.0.0") - # non-Apple Clang. + +def test_clang_flags(): supported_flag_test("version_argument", "--version", "clang@foo.bar") supported_flag_test("openmp_flag", "-fopenmp", "clang@3.3") unsupported_flag_test("cxx11_flag", "clang@3.2") @@ -713,3 +717,101 @@ def _call(*args, **kwargs): except ProcessError: # Confirm environment does not change after failed call assert 'SPACK_TEST_CMP_ON' not in os.environ + + +def test_apple_clang_setup_environment(mock_executable, monkeypatch): + """Test a code path that is taken only if the package uses + Xcode on MacOS. + """ + class MockPackage(object): + use_xcode = False + + apple_clang_cls = spack.compilers.class_for_compiler_name('apple-clang') + compiler = apple_clang_cls( + spack.spec.CompilerSpec('apple-clang@11.0.0'), 'catalina', 'x86_64', [ + '/usr/bin/clang', '/usr/bin/clang++', None, None + ] + ) + env = spack.util.environment.EnvironmentModifications() + # Check a package that doesn't use xcode and ensure we don't add changes + # to the environment + pkg = MockPackage() + compiler.setup_custom_environment(pkg, env) + assert not env + + # Prepare mock executables to fake the Xcode environment + xcrun = mock_executable('xcrun', """ +if [[ "$2" == "clang" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang" +fi +if [[ "$2" == "clang++" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang++" +fi +""") + mock_executable('xcode-select', """ +echo "/Library/Developer" +""") + bin_dir = os.path.dirname(xcrun) + monkeypatch.setenv('PATH', bin_dir, prepend=os.pathsep) + + def noop(*args, **kwargs): + pass + + real_listdir = os.listdir + + def _listdir(path): + if not os.path.exists(path): + return [] + return real_listdir(path) + + # Set a few operations to noop + monkeypatch.setattr(shutil, 'copytree', noop) + monkeypatch.setattr(os, 'unlink', noop) + monkeypatch.setattr(os, 'symlink', noop) + monkeypatch.setattr(os, 'listdir', _listdir) + + # Qt is so far the only package that uses this code path, change + # introduced in https://github.com/spack/spack/pull/1832 + pkg.use_xcode = True + compiler.setup_custom_environment(pkg, env) + assert len(env) == 3 + assert env.env_modifications[0].name == 'SPACK_CC' + assert env.env_modifications[1].name == 'SPACK_CXX' + assert env.env_modifications[2].name == 'DEVELOPER_DIR' + + +@pytest.mark.parametrize('xcode_select_output', [ + '', '/Library/Developer/CommandLineTools' +]) +def test_xcode_not_available( + xcode_select_output, mock_executable, monkeypatch +): + # Prepare mock executables to fake the Xcode environment + xcrun = mock_executable('xcrun', """ + if [[ "$2" == "clang" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang" + fi + if [[ "$2" == "clang++" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang++" + fi + """) + mock_executable('xcode-select', """ + echo "{0}" + """.format(xcode_select_output)) + bin_dir = os.path.dirname(xcrun) + monkeypatch.setenv('PATH', bin_dir, prepend=os.pathsep) + # Prepare compiler + apple_clang_cls = spack.compilers.class_for_compiler_name('apple-clang') + compiler = apple_clang_cls( + spack.spec.CompilerSpec('apple-clang@11.0.0'), 'catalina', 'x86_64', [ + '/usr/bin/clang', '/usr/bin/clang++', None, None + ] + ) + env = spack.util.environment.EnvironmentModifications() + + class MockPackage(object): + use_xcode = True + + pkg = MockPackage() + with pytest.raises(OSError): + compiler.setup_custom_environment(pkg, env) diff --git a/lib/spack/spack/test/compilers/detection.py b/lib/spack/spack/test/compilers/detection.py index 90311ad2d32..b38cf89b5dd 100644 --- a/lib/spack/spack/test/compilers/detection.py +++ b/lib/spack/spack/test/compilers/detection.py @@ -53,11 +53,22 @@ def test_cce_version_detection(version_str, expected_version): 'Target: x86_64-apple-darwin18.7.0\n' 'Thread model: posix\n' 'InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin\n', # noqa - '11.0.0-apple'), + '11.0.0'), ('Apple LLVM version 7.0.2 (clang-700.1.81)\n' 'Target: x86_64-apple-darwin15.2.0\n' - 'Thread model: posix\n', '7.0.2-apple'), - # Other platforms + 'Thread model: posix\n', '7.0.2'), +]) +def test_apple_clang_version_detection( + version_str, expected_version +): + cls = spack.compilers.class_for_compiler_name('apple-clang') + version = cls.extract_version_from_output(version_str) + assert version == expected_version + + +@pytest.mark.regression('10191') +@pytest.mark.parametrize('version_str,expected_version', [ + # LLVM Clang ('clang version 6.0.1-svn334776-1~exp1~20181018152737.116 (branches/release_60)\n' # noqa 'Target: x86_64-pc-linux-gnu\n' 'Thread model: posix\n' diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index fd0e1851685..cfe0748c4bf 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -608,7 +608,7 @@ def test_noversion_pkg(self, spec): ('mpileaks%gcc@4.8', 'haswell'), ('mpileaks%gcc@5.3.0', 'broadwell'), # Apple's clang always falls back to x86-64 for now - ('mpileaks%clang@9.1.0-apple', 'x86_64') + ('mpileaks%apple-clang@9.1.0', 'x86_64') ]) @pytest.mark.regression('13361') def test_adjusting_default_target_based_on_compiler( diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 1efdd6ba959..9a292821f5a 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1145,7 +1145,7 @@ def mock_executable(tmpdir): import jinja2 def _factory(name, output, subdir=('bin',)): - f = tmpdir.mkdir(*subdir).join(name) + f = tmpdir.ensure(*subdir, dir=True).join(name) t = jinja2.Template('#!/bin/bash\n{{ output }}\n') f.write(t.render(output=output)) f.chmod(0o755) diff --git a/lib/spack/spack/test/data/config/compilers.yaml b/lib/spack/spack/test/data/config/compilers.yaml index 7aec1384736..3a2db05e72f 100644 --- a/lib/spack/spack/test/data/config/compilers.yaml +++ b/lib/spack/spack/test/data/config/compilers.yaml @@ -127,7 +127,7 @@ compilers: cxxflags: -O3 modules: 'None' - compiler: - spec: clang@9.1.0-apple + spec: apple-clang@9.1.0 operating_system: elcapitan paths: cc: /path/to/clang diff --git a/var/spack/repos/builtin/packages/allpaths-lg/package.py b/var/spack/repos/builtin/packages/allpaths-lg/package.py index 6ebca2eee62..b0bb49b3a19 100644 --- a/var/spack/repos/builtin/packages/allpaths-lg/package.py +++ b/var/spack/repos/builtin/packages/allpaths-lg/package.py @@ -18,6 +18,7 @@ class AllpathsLg(AutotoolsPackage): # compiles with gcc 4.7.0 to 4.9.4) conflicts('%gcc@:4.6.4,5.1.0:') conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%intel') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/aluminum/package.py b/var/spack/repos/builtin/packages/aluminum/package.py index 5edc4c37927..c5d6a45ed8b 100644 --- a/var/spack/repos/builtin/packages/aluminum/package.py +++ b/var/spack/repos/builtin/packages/aluminum/package.py @@ -47,8 +47,8 @@ def cmake_args(self): '-DALUMINUM_ENABLE_MPI_CUDA:BOOL=%s' % ('+mpi_cuda' in spec), '-DALUMINUM_ENABLE_NCCL:BOOL=%s' % ('+nccl' in spec)] - # Add support for OS X to find OpenMP - if (self.spec.satisfies('%clang platform=darwin')): + # Add support for OS X to find OpenMP (LLVM installed via brew) + if self.spec.satisfies('%clang platform=darwin'): clang = self.compiler.cc clang_bin = os.path.dirname(clang) clang_root = os.path.dirname(clang_bin) diff --git a/var/spack/repos/builtin/packages/amber/package.py b/var/spack/repos/builtin/packages/amber/package.py index d2ec13fb3e9..481377047ca 100644 --- a/var/spack/repos/builtin/packages/amber/package.py +++ b/var/spack/repos/builtin/packages/amber/package.py @@ -100,12 +100,10 @@ class Amber(Package, CudaPackage): depends_on('cuda@7.5.18', when='@:16+cuda') # conflicts - conflicts('+x11', when='platform=cray', - msg='x11 amber applications not available for cray') - conflicts('+openmp', when='%clang', - msg='openmp optimizations not available for the clang compiler') - conflicts('+openmp', when='%pgi', - msg='openmp optimizations not available for the pgi compiler') + conflicts('+x11', when='platform=cray', msg='x11 amber applications not available for cray') + conflicts('+openmp', when='%clang', msg='OpenMP optimizations not available for the clang compiler') + conflicts('+openmp', when='%apple-clang', msg='OpenMP optimizations not available for the Apple clang compiler') + conflicts('+openmp', when='%pgi', msg='OpenMP optimizations not available for the pgi compiler') def setup_build_environment(self, env): amber_src = self.stage.source_path diff --git a/var/spack/repos/builtin/packages/amrex/package.py b/var/spack/repos/builtin/packages/amrex/package.py index 32da808f1d1..f6cb778f4b4 100644 --- a/var/spack/repos/builtin/packages/amrex/package.py +++ b/var/spack/repos/builtin/packages/amrex/package.py @@ -65,6 +65,7 @@ class Amrex(CMakePackage): depends_on('cmake@3.5:', type='build', when='@:18.10.99') depends_on('cmake@3.13:', type='build', when='@18.11:') depends_on('cmake@3.14:', type='build', when='@19.04:') + conflicts('%apple-clang') conflicts('%clang') def url_for_version(self, version): diff --git a/var/spack/repos/builtin/packages/ape/package.py b/var/spack/repos/builtin/packages/ape/package.py index 114185aaa4a..bf9f895e689 100644 --- a/var/spack/repos/builtin/packages/ape/package.py +++ b/var/spack/repos/builtin/packages/ape/package.py @@ -29,11 +29,13 @@ def install(self, spec, prefix): # When preprocessor expands macros (i.e. CFLAGS) defined as quoted # strings the result may be > 132 chars and is terminated. # This will look to a compiler as an Unterminated character constant - # and produce Line truncated errors. To vercome this, add flags to + # and produce Line truncated errors. To overcome this, add flags to # let compiler know that the entire line is meaningful. # TODO: For the lack of better approach, assume that clang is mixed - # with GNU fortran. - if spec.satisfies('%clang') or spec.satisfies('%gcc'): + # TODO: with GNU fortran. + if (spec.satisfies('%apple-clang') or + spec.satisfies('%clang') or + spec.satisfies('%gcc')): args.extend([ 'FCFLAGS=-O2 -ffree-line-length-none' ]) diff --git a/var/spack/repos/builtin/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py index 4d56d0016db..2a355dfca78 100644 --- a/var/spack/repos/builtin/packages/atlas/package.py +++ b/var/spack/repos/builtin/packages/atlas/package.py @@ -102,7 +102,7 @@ def install(self, spec, prefix): # Workaround for macOS Clang: # http://math-atlas.sourceforge.net/atlas_install/node66.html - if spec.satisfies('@3.10.3: %clang platform=darwin'): + if spec.satisfies('@3.10.3: %apple-clang'): options.append('--force-clang=' + spack_cc) # Lapack resource to provide full lapack build. Note that diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 791606f5645..96fdcc067b3 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -120,9 +120,10 @@ def install_headers(self): def flag_handler(self, name, flags): # To ignore the errors of narrowing conversions for # the Fujitsu compiler - if name == 'cxxflags'\ - and (self.compiler.name == 'fj' or self.compiler.name == 'clang')\ - and self.version <= ver('2.31.1'): + if name == 'cxxflags' and ( + self.spec.satisfies('@:2.31.1') and + self.compiler.name in ('fj', 'clang', 'apple-clang') + ): flags.append('-Wno-narrowing') elif name == 'cflags': if self.spec.satisfies('@:2.34 %gcc@10:'): diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 1e5bd00c2bd..d0d8691c33f 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -398,7 +398,9 @@ def determine_b2_options(self, spec, options): # and at least in clang 3.9 still fails to build # http://www.boost.org/build/doc/html/bbv2/reference/precompiled_headers.html # https://svn.boost.org/trac/boost/ticket/12496 - if spec.satisfies('%clang') or spec.satisfies('%fj'): + if (spec.satisfies('%apple-clang') or + spec.satisfies('%clang') or + spec.satisfies('%fj')): options.extend(['pch=off']) if '+clanglibcpp' in spec: cxxflags.append('-stdlib=libc++') diff --git a/var/spack/repos/builtin/packages/bucky/package.py b/var/spack/repos/builtin/packages/bucky/package.py index 859fabde347..41337e7e844 100644 --- a/var/spack/repos/builtin/packages/bucky/package.py +++ b/var/spack/repos/builtin/packages/bucky/package.py @@ -19,6 +19,7 @@ class Bucky(MakefilePackage): # Compilation requires gcc conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%intel') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/cbench/package.py b/var/spack/repos/builtin/packages/cbench/package.py index ceeec3b468e..8a24a54d6ea 100644 --- a/var/spack/repos/builtin/packages/cbench/package.py +++ b/var/spack/repos/builtin/packages/cbench/package.py @@ -26,6 +26,7 @@ class Cbench(MakefilePackage): # The following compilers are not supported by Cbench: conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%nag') conflicts('%xl') diff --git a/var/spack/repos/builtin/packages/cp2k/package.py b/var/spack/repos/builtin/packages/cp2k/package.py index 0a172d025fe..1fba619ffc2 100644 --- a/var/spack/repos/builtin/packages/cp2k/package.py +++ b/var/spack/repos/builtin/packages/cp2k/package.py @@ -141,6 +141,7 @@ class Cp2k(MakefilePackage, CudaPackage): depends_on('wannier90', when='@3.0+mpi', type='build') # CP2K needs compiler specific compilation flags, e.g. optflags + conflicts('%apple-clang') conflicts('%clang') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 0f46c318f6d..8d7bd2be815 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -301,7 +301,7 @@ def cmake_args(self): cxx_flags_release.extend(['-O3']) elif spec.satisfies('%intel'): cxx_flags_release.extend(['-O3']) - elif spec.satisfies('%clang'): + elif spec.satisfies('%clang') or spec.satisfies('%apple-clang'): cxx_flags_release.extend(['-O3', '-ffp-contract=fast']) # Python bindings diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py index 651caf3d76d..cd6b9612270 100644 --- a/var/spack/repos/builtin/packages/elfutils/package.py +++ b/var/spack/repos/builtin/packages/elfutils/package.py @@ -56,6 +56,7 @@ class Elfutils(AutotoolsPackage, SourcewarePackage): # in gcc, but not in clang. C code compiled with gcc is # binary-compatible with clang, so it should be possible to build # elfutils with gcc, and then link it to clang-built libraries. + conflicts('%apple-clang') conflicts('%clang') # Elfutils uses -Wall and we don't want to fail the build over a diff --git a/var/spack/repos/builtin/packages/f18/package.py b/var/spack/repos/builtin/packages/f18/package.py index 699cfde6c68..235210c142e 100644 --- a/var/spack/repos/builtin/packages/f18/package.py +++ b/var/spack/repos/builtin/packages/f18/package.py @@ -30,6 +30,9 @@ class F18(CMakePackage): # Conflicts compiler_warning = 'F18 requires a compiler with support for C++17' + # See https://en.wikipedia.org/wiki/Xcode#Latest_versions for a + # conversion table from LLVM versions to Apple's Clang + conflicts('%apple-clang@:10.1', msg=compiler_warning) conflicts('%clang@:6', msg=compiler_warning) conflicts('%gcc@:7.1', msg=compiler_warning) conflicts('%intel', msg=compiler_warning) diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 961c4cfc661..b644677fd1a 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -57,6 +57,7 @@ class Fftw(AutotoolsPackage): msg='Long double precision is not supported in FFTW 2') conflicts('precision=quad', when='@2.1.5', msg='Quad precision is not supported in FFTW 2') + conflicts('+openmp', when='%apple-clang', msg="Apple's clang does not support OpenMP") provides('fftw-api@2', when='@2.1.5') provides('fftw-api@3', when='@3:') @@ -124,11 +125,6 @@ def configure(self, spec, prefix): # Variants that affect every precision if '+openmp' in spec: - # Note: Apple's Clang does not support OpenMP. - if spec.satisfies('%clang'): - ver = str(self.compiler.version) - if ver.endswith('-apple'): - raise InstallError("Apple's clang does not support OpenMP") options.append('--enable-openmp') if spec.satisfies('@:2'): # TODO: libtool strips CFLAGS, so 2.x libxfftw_threads diff --git a/var/spack/repos/builtin/packages/fujitsu-mpi/package.py b/var/spack/repos/builtin/packages/fujitsu-mpi/package.py index c0a882ec4c7..fcf4dc8e94b 100644 --- a/var/spack/repos/builtin/packages/fujitsu-mpi/package.py +++ b/var/spack/repos/builtin/packages/fujitsu-mpi/package.py @@ -13,6 +13,7 @@ class FujitsuMpi(Package): conflicts('%arm') conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%gcc') conflicts('%intel') diff --git a/var/spack/repos/builtin/packages/gitconddb/package.py b/var/spack/repos/builtin/packages/gitconddb/package.py index 9400ea549ce..e2808e91c03 100644 --- a/var/spack/repos/builtin/packages/gitconddb/package.py +++ b/var/spack/repos/builtin/packages/gitconddb/package.py @@ -40,4 +40,5 @@ class Gitconddb(CMakePackage): # Known conflicts on C++17 compatibility (aggressive for now) conflicts('%gcc@:7.9.999', msg="GitCondDB requires GCC 8 or newer for C++17 support") + conflicts('%apple-clang', when="@:0.1.99", msg="No Darwin support for clang in older versions") conflicts('%clang platform=darwin', when="@:0.1.99", msg="No Darwin support for clang in older versions") diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py index 4f8e772f19d..34b2d8dd2e8 100644 --- a/var/spack/repos/builtin/packages/graphviz/package.py +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -137,6 +137,13 @@ def patch(self): filter_file(r'-lstdc\+\+', '-lc++', 'configure.ac', *(d + '/Makefile.am' for d in mkdirs)) + @when('%apple-clang') + def patch(self): + # When using Clang, replace GCC's libstdc++ with LLVM's libc++ + mkdirs = ['cmd/dot', 'cmd/edgepaint', 'cmd/mingle', 'plugin/gdiplus'] + filter_file(r'-lstdc\+\+', '-lc++', 'configure.ac', + *(d + '/Makefile.am' for d in mkdirs)) + def configure_args(self): spec = self.spec args = ['--disable-silent-rules'] diff --git a/var/spack/repos/builtin/packages/hpgmg/package.py b/var/spack/repos/builtin/packages/hpgmg/package.py index e701d1c212e..f5567ba2c92 100644 --- a/var/spack/repos/builtin/packages/hpgmg/package.py +++ b/var/spack/repos/builtin/packages/hpgmg/package.py @@ -59,13 +59,11 @@ def configure_args(self): args.append('--no-fv') else: # Apple's Clang doesn't support OpenMP - if not (self.spec.satisfies('%clang') and self.compiler.is_apple): + if not self.spec.satisfies('%apple-clang'): cflags.append(self.compiler.openmp_flag) if '+debug' in self.spec: cflags.append('-g') - elif any(map(self.spec.satisfies, ['%gcc', '%clang', '%intel'])): - cflags.append('-O3') else: cflags.append('-O3') diff --git a/var/spack/repos/builtin/packages/hydrogen/package.py b/var/spack/repos/builtin/packages/hydrogen/package.py index 449fe046dc5..9d12235d720 100644 --- a/var/spack/repos/builtin/packages/hydrogen/package.py +++ b/var/spack/repos/builtin/packages/hydrogen/package.py @@ -131,8 +131,8 @@ def cmake_args(self): '-DHydrogen_ENABLE_HALF=%s' % ('+half' in spec), ] - # Add support for OS X to find OpenMP - if (self.spec.satisfies('%clang platform=darwin')): + # Add support for OS X to find OpenMP (LLVM installed via brew) + if self.spec.satisfies('%clang platform=darwin'): clang = self.compiler.cc clang_bin = os.path.dirname(clang) clang_root = os.path.dirname(clang_bin) diff --git a/var/spack/repos/builtin/packages/intel-tbb/package.py b/var/spack/repos/builtin/packages/intel-tbb/package.py index c470e8933b3..4c652f6fce3 100644 --- a/var/spack/repos/builtin/packages/intel-tbb/package.py +++ b/var/spack/repos/builtin/packages/intel-tbb/package.py @@ -69,6 +69,8 @@ class IntelTbb(Package): # # See https://github.com/intel/tbb/pull/147 for details. # + conflicts('%apple-clang', when='@:2019.6', + msg='2019.7 or later required for clang') conflicts('%clang', when='@:2019.6', msg='2019.7 or later required for clang') @@ -163,7 +165,7 @@ def install(self, spec, prefix): # self.coerce_to_spack("build") - if spec.satisfies('%clang'): + if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): tbb_compiler = "clang" elif spec.satisfies('%intel'): tbb_compiler = "icc" diff --git a/var/spack/repos/builtin/packages/kahip/package.py b/var/spack/repos/builtin/packages/kahip/package.py index 7d12d36bc2e..e2ca1aeaa1c 100644 --- a/var/spack/repos/builtin/packages/kahip/package.py +++ b/var/spack/repos/builtin/packages/kahip/package.py @@ -32,6 +32,7 @@ class Kahip(SConsPackage): depends_on('argtable') depends_on('mpi') # Note: upstream package only tested on openmpi + conflicts('%apple-clang') conflicts('%clang') def patch(self): diff --git a/var/spack/repos/builtin/packages/kentutils/package.py b/var/spack/repos/builtin/packages/kentutils/package.py index 379d3486c03..f0b4ee83221 100644 --- a/var/spack/repos/builtin/packages/kentutils/package.py +++ b/var/spack/repos/builtin/packages/kentutils/package.py @@ -22,6 +22,7 @@ class Kentutils(MakefilePackage): depends_on('mariadb') conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%intel') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/lbann/package.py b/var/spack/repos/builtin/packages/lbann/package.py index 04b044610fa..bc0da109ec5 100644 --- a/var/spack/repos/builtin/packages/lbann/package.py +++ b/var/spack/repos/builtin/packages/lbann/package.py @@ -164,16 +164,16 @@ def cmake_args(self): # protobuf is included by py-protobuf+cpp '-DProtobuf_DIR={0}'.format(spec['protobuf'].prefix)]) - if self.spec.satisfies('@:0.90') or self.spec.satisfies('@0.95:'): + if spec.satisfies('@:0.90') or spec.satisfies('@0.95:'): args.extend([ '-DHydrogen_DIR={0}/CMake/hydrogen'.format( spec['hydrogen'].prefix)]) - elif self.spec.satisfies('@0.94'): + elif spec.satisfies('@0.94'): args.extend([ '-DElemental_DIR={0}/CMake/elemental'.format( spec['elemental'].prefix)]) - if self.spec.satisfies('@0.94:0.98.2'): + if spec.satisfies('@0.94:0.98.2'): args.extend(['-DLBANN_WITH_NCCL:BOOL=%s' % ('+gpu +nccl' in spec)]) if '+vtune' in spec: @@ -188,8 +188,8 @@ def cmake_args(self): '-DConduit_DIR={0}'.format(spec['conduit'].prefix)]) # Add support for OpenMP - if (self.spec.satisfies('%clang')): - if (sys.platform == 'darwin'): + if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): + if sys.platform == 'darwin': clang = self.compiler.cc clang_bin = os.path.dirname(clang) clang_root = os.path.dirname(clang_bin) @@ -210,7 +210,7 @@ def cmake_args(self): args.extend([ '-DcuDNN_DIR={0}'.format( spec['cudnn'].prefix)]) - if self.spec.satisfies('@0.94:0.98.2'): + if spec.satisfies('@0.94:0.98.2'): args.extend(['-DCUB_DIR={0}'.format( spec['cub'].prefix)]) if '+nccl' in spec: diff --git a/var/spack/repos/builtin/packages/libceed/package.py b/var/spack/repos/builtin/packages/libceed/package.py index 36d46dae0bc..a7ae5b21f86 100644 --- a/var/spack/repos/builtin/packages/libceed/package.py +++ b/var/spack/repos/builtin/packages/libceed/package.py @@ -71,14 +71,14 @@ def common_make_opts(self): opt = '-O3 -g -ffp-contract=fast' if compiler.version >= ver(4.9): opt += ' -fopenmp-simd' + elif compiler.name == 'apple-clang': + opt = '-O3 -g -march=native -ffp-contract=fast' + if compiler.version >= ver(10): + opt += ' -fopenmp-simd' elif compiler.name == 'clang': opt = '-O3 -g -march=native -ffp-contract=fast' - if compiler.version.string.endswith('-apple'): - if compiler.version >= ver(10): - opt += ' -fopenmp-simd' - else: # not apple clang - if compiler.version >= ver(6): - opt += ' -fopenmp-simd' + if compiler.version >= ver(6): + opt += ' -fopenmp-simd' elif compiler.name in ['xl', 'xl_r']: opt = '-O -g -qsimd=auto' else: diff --git a/var/spack/repos/builtin/packages/libcerf/package.py b/var/spack/repos/builtin/packages/libcerf/package.py index e37e7f63399..a5c00dabaf0 100644 --- a/var/spack/repos/builtin/packages/libcerf/package.py +++ b/var/spack/repos/builtin/packages/libcerf/package.py @@ -23,7 +23,7 @@ def configure_args(self): options = [] # Clang reports unused functions as errors, see # http://clang.debian.net/status.php?version=3.8.1&key=UNUSED_FUNCTION - if spec.satisfies('%clang'): + if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): options.append('CFLAGS=-Wno-unused-function') # fujitsu compiler has a error about unused functions too. if spec.satisfies('%fj'): diff --git a/var/spack/repos/builtin/packages/librsb/package.py b/var/spack/repos/builtin/packages/librsb/package.py index 90cc5b67a82..2a292b94798 100644 --- a/var/spack/repos/builtin/packages/librsb/package.py +++ b/var/spack/repos/builtin/packages/librsb/package.py @@ -17,6 +17,7 @@ class Librsb(AutotoolsPackage): version('1.2.0.8', '8bebd19a1866d80ade13eabfdd0f07ae7e8a485c0b975b5d15f531ac204d80cb') depends_on('zlib') + conflicts('%apple-clang') conflicts('%clang') def configure_args(self): diff --git a/var/spack/repos/builtin/packages/likwid/package.py b/var/spack/repos/builtin/packages/likwid/package.py index 8e89693ce1b..a104b670036 100644 --- a/var/spack/repos/builtin/packages/likwid/package.py +++ b/var/spack/repos/builtin/packages/likwid/package.py @@ -60,9 +60,15 @@ def filter_sbang(self): *files) def install(self, spec, prefix): - supported_compilers = {'clang': 'CLANG', 'gcc': 'GCC', 'intel': 'ICC'} + supported_compilers = { + 'apple-clang': 'CLANG', + 'clang': 'CLANG', + 'gcc': 'GCC', + 'intel': 'ICC' + } if spec.target.family == 'aarch64': - supported_compilers = {'gcc': 'GCCARMv8', 'clang': 'ARMCLANG'} + supported_compilers = { + 'gcc': 'GCCARMv8', 'clang': 'ARMCLANG', 'arm': 'ARMCLANG'} elif spec.target.family == 'ppc64' or spec.target.family == 'ppc64le': supported_compilers = {'gcc': 'GCCPOWER'} if self.compiler.name not in supported_compilers: diff --git a/var/spack/repos/builtin/packages/maverick/package.py b/var/spack/repos/builtin/packages/maverick/package.py index 6c560bb202d..60909c551bb 100644 --- a/var/spack/repos/builtin/packages/maverick/package.py +++ b/var/spack/repos/builtin/packages/maverick/package.py @@ -17,6 +17,7 @@ class Maverick(MakefilePackage): conflicts('%gcc@:6.0') conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%intel') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c9c7886a088..d862f125224 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -79,6 +79,7 @@ class Mpich(AutotoolsPackage): # see https://lists.mpich.org/pipermail/discuss/2016-May/004764.html # and https://lists.mpich.org/pipermail/discuss/2016-June/004768.html patch('mpich32_clang.patch', when='@3.2:3.2.0%clang') + patch('mpich32_clang.patch', when='@3.2:3.2.0%apple-clang') # Fix SLURM node list parsing # See https://github.com/pmodels/mpich/issues/3572 diff --git a/var/spack/repos/builtin/packages/mpilander/package.py b/var/spack/repos/builtin/packages/mpilander/package.py index 276f380a868..9b729151807 100644 --- a/var/spack/repos/builtin/packages/mpilander/package.py +++ b/var/spack/repos/builtin/packages/mpilander/package.py @@ -31,6 +31,7 @@ class Mpilander(CMakePackage): # compiler support conflicts('%gcc@:4.7') conflicts('%clang@:3.8') + conflicts('%apple-clang@:7.4') conflicts('%intel@:16') def cmake_args(self): diff --git a/var/spack/repos/builtin/packages/musl/package.py b/var/spack/repos/builtin/packages/musl/package.py index 7c68735c200..05ec4eb5ff3 100644 --- a/var/spack/repos/builtin/packages/musl/package.py +++ b/var/spack/repos/builtin/packages/musl/package.py @@ -38,7 +38,7 @@ def patch(self): if self.compiler.name == 'gcc': config.filter("WRAPCC_GCC = .*'", "WRAPCC_GCC = {0}'". format(self.compiler.cc)) - elif self.compiler.name == 'clang': + elif self.compiler.name in ('clang', 'apple-clang'): config.filter("WRAPCC_CLANG = .*'", "WRAPCC_CLANG = {0}'". format(self.compiler.cc)) @@ -46,7 +46,7 @@ def configure_args(self): args = ['--prefix={0}'.format(self.prefix)] if self.compiler.name == 'gcc': args.append('--enable-wrapper=gcc') - elif self.compiler.name == 'clang': + elif self.compiler.name in ('clang', 'apple-clang'): args.append('--enable-wrapper=clang') else: args.append('--enable-wrapper=no') diff --git a/var/spack/repos/builtin/packages/octopus/package.py b/var/spack/repos/builtin/packages/octopus/package.py index 1073965e817..bed1c1a2569 100644 --- a/var/spack/repos/builtin/packages/octopus/package.py +++ b/var/spack/repos/builtin/packages/octopus/package.py @@ -97,11 +97,13 @@ def install(self, spec, prefix): # When preprocessor expands macros (i.e. CFLAGS) defined as quoted # strings the result may be > 132 chars and is terminated. # This will look to a compiler as an Unterminated character constant - # and produce Line truncated errors. To vercome this, add flags to + # and produce Line truncated errors. To overcome this, add flags to # let compiler know that the entire line is meaningful. # TODO: For the lack of better approach, assume that clang is mixed # with GNU fortran. - if spec.satisfies('%clang') or spec.satisfies('%gcc'): + if (spec.satisfies('%apple-clang') or + spec.satisfies('%clang') or + spec.satisfies('%gcc')): args.extend([ 'FCFLAGS=-O2 -ffree-line-length-none' ]) diff --git a/var/spack/repos/builtin/packages/onednn/package.py b/var/spack/repos/builtin/packages/onednn/package.py index b723fabd9c9..82d22296f9d 100644 --- a/var/spack/repos/builtin/packages/onednn/package.py +++ b/var/spack/repos/builtin/packages/onednn/package.py @@ -64,7 +64,7 @@ class Onednn(CMakePackage): # https://github.com/oneapi-src/oneDNN#requirements-for-building-from-source depends_on('cmake@2.8.11:', type='build') depends_on('tbb@2017:', when='cpu_runtime=tbb') - depends_on('llvm-openmp', when='%clang platform=darwin cpu_runtime=omp') + depends_on('llvm-openmp', when='%apple-clang cpu_runtime=omp') depends_on('opencl@1.2:', when='gpu_runtime=ocl') def cmake_args(self): @@ -81,7 +81,7 @@ def cmake_args(self): args.append('-DDNNL_BUILD_TESTS=OFF') # https://github.com/oneapi-src/oneDNN/issues/591 - if self.spec.satisfies('%clang platform=darwin cpu_runtime=omp'): + if self.spec.satisfies('%apple-clang cpu_runtime=omp'): args.extend([ '-DOpenMP_CXX_FLAGS={0}'.format(self.compiler.openmp_flag), '-DOpenMP_C_FLAGS={0}'.format(self.compiler.openmp_flag), diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index c1a0c1e01bd..61a530b37f4 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -95,7 +95,7 @@ class Openblas(MakefilePackage): patch('openblas-0.3.8-darwin.patch', when='@0.3.8 platform=darwin') # Fix ICE in LLVM 9.0.0 https://github.com/xianyi/OpenBLAS/pull/2329 # Patch as in https://github.com/xianyi/OpenBLAS/pull/2597 - patch('openblas_appleclang11.patch', when='@0.3.8:0.3.9 %clang@11.0.3-apple') + patch('openblas_appleclang11.patch', when='@0.3.8:0.3.9 %apple-clang@11.0.3') # Add conditions to f_check to determine the Fujitsu compiler patch('openblas_fujitsu.patch', when='%fj') @@ -105,6 +105,9 @@ class Openblas(MakefilePackage): conflicts('+consistent_fpcsr', when='threads=none', msg='FPCSR consistency only applies to multithreading') + conflicts('threads=openmp', when='%apple-clang', msg="Apple's clang does not support OpenMP") + conflicts('threads=openmp @:0.2.19', when='%clang', msg='OpenBLAS @:0.2.19 does not support OpenMP with clang!') + @property def parallel(self): # unclear whether setting `-j N` externally was supported before 0.3 @@ -120,18 +123,6 @@ def check_compilers(self): 'OpenBLAS requires both C and Fortran compilers!' ) - # Add support for OpenMP - if (self.spec.satisfies('threads=openmp') and - self.spec.satisfies('%clang')): - if str(self.spec.compiler.version).endswith('-apple'): - raise InstallError("Apple's clang does not support OpenMP") - if '@:0.2.19' in self.spec: - # Openblas (as of 0.2.19) hardcoded that OpenMP cannot - # be used with any (!) compiler named clang, bummer. - raise InstallError( - 'OpenBLAS @:0.2.19 does not support OpenMP with clang!' - ) - @staticmethod def _read_targets(target_file): """Parse a list of available targets from the OpenBLAS/TargetList.txt diff --git a/var/spack/repos/builtin/packages/pdt/package.py b/var/spack/repos/builtin/packages/pdt/package.py index 35164db1c55..bf3f954982c 100644 --- a/var/spack/repos/builtin/packages/pdt/package.py +++ b/var/spack/repos/builtin/packages/pdt/package.py @@ -33,7 +33,8 @@ class Pdt(AutotoolsPackage): variant('pic', default=False, description="Builds with pic") def patch(self): - if self.spec.satisfies('%clang'): + spec = self.spec + if spec.satisfies('%clang') or spec.satisfies('%apple-clang'): filter_file(r'PDT_GXX=g\+\+ ', r'PDT_GXX=clang++ ', 'ductape/Makefile') diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index f328171072e..9da97406746 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -2,11 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - - import os -import sys -from spack import * class Petsc(Package): @@ -116,9 +112,7 @@ class Petsc(Package): # temporary workaround Clang 8.1.0 with XCode 8.3 on macOS, see # https://bitbucket.org/petsc/petsc/commits/4f290403fdd060d09d5cb07345cbfd52670e3cbc # the patch is an adaptation of the original commit to 3.7.5 - if sys.platform == "darwin": - patch('macos-clang-8.1.0.diff', - when='@3.7.5%clang@8.1.0:') + patch('macos-clang-8.1.0.diff', when='@3.7.5%apple-clang@8.1.0:') patch('pkg-config-3.7.6-3.8.4.diff', when='@3.7.6:3.8.4') patch('xcode_stub_out_of_sync.patch', when='@:3.10.4') diff --git a/var/spack/repos/builtin/packages/pixman/package.py b/var/spack/repos/builtin/packages/pixman/package.py index 1087dabbbff..a145edc638d 100644 --- a/var/spack/repos/builtin/packages/pixman/package.py +++ b/var/spack/repos/builtin/packages/pixman/package.py @@ -29,7 +29,8 @@ class Pixman(AutotoolsPackage): # __builtin_shuffle was removed in clang 5.0. # From version 9.1 apple-clang is based on clang 5.0. # Patch is obtained from above link. - patch('clang.patch', when='@0.34%clang@9.1.0-apple:') + patch('clang.patch', when='@0.34%apple-clang@9.1.0:') + patch('clang.patch', when='@0.34%clang@5.0.0:') @run_before('build') def patch_config_h_for_intel(self): diff --git a/var/spack/repos/builtin/packages/plasma/package.py b/var/spack/repos/builtin/packages/plasma/package.py index bf4d8c68ee9..efeb980bd86 100644 --- a/var/spack/repos/builtin/packages/plasma/package.py +++ b/var/spack/repos/builtin/packages/plasma/package.py @@ -58,6 +58,7 @@ class Plasma(CMakePackage): conflicts("%gcc@:5.99", when='@17.2:') conflicts("%cce") + conflicts('%apple-clang') conflicts("%clang") conflicts("%intel") conflicts("%nag") diff --git a/var/spack/repos/builtin/packages/precice/package.py b/var/spack/repos/builtin/packages/precice/package.py index 11250f60bda..45548275c89 100644 --- a/var/spack/repos/builtin/packages/precice/package.py +++ b/var/spack/repos/builtin/packages/precice/package.py @@ -60,6 +60,7 @@ class Precice(CMakePackage): # We require C++11 compiler support as well as # library support for time manipulators (N2071, N2072) conflicts('%gcc@:4') + conflicts('%apple-clang@:4') conflicts('%clang@:3.7') conflicts('%intel@:14') conflicts('%pgi@:14') diff --git a/var/spack/repos/builtin/packages/py-dgl/package.py b/var/spack/repos/builtin/packages/py-dgl/package.py index 544b527f69b..9ef1bd9ed86 100644 --- a/var/spack/repos/builtin/packages/py-dgl/package.py +++ b/var/spack/repos/builtin/packages/py-dgl/package.py @@ -29,7 +29,7 @@ class PyDgl(CMakePackage): depends_on('cmake@3.5:', type='build') depends_on('cuda', when='+cuda') - depends_on('llvm-openmp', when='%clang platform=darwin +openmp') + depends_on('llvm-openmp', when='%apple-clang +openmp') # Python dependencies # See python/setup.py @@ -66,7 +66,7 @@ def cmake_args(self): if '+openmp' in self.spec: args.append('-DUSE_OPENMP=ON') - if self.spec.satisfies('%clang platform=darwin'): + if self.spec.satisfies('%apple-clang'): args.extend([ '-DOpenMP_CXX_FLAGS=' + self.spec['llvm-openmp'].headers.include_flags, diff --git a/var/spack/repos/builtin/packages/py-exodus-bundler/package.py b/var/spack/repos/builtin/packages/py-exodus-bundler/package.py index c13f30eb469..eddb3da88fa 100644 --- a/var/spack/repos/builtin/packages/py-exodus-bundler/package.py +++ b/var/spack/repos/builtin/packages/py-exodus-bundler/package.py @@ -15,6 +15,7 @@ class PyExodusBundler(PythonPackage): version('2.0.2', sha256='4e896a2034b94cf7b4fb33d86a68e29a7d3b08e57541e444db34dddc6ac1ef68') + depends_on('musl', type='run', when='%apple-clang') depends_on('musl', type='run', when='%clang') depends_on('musl', type='run', when='%gcc') depends_on('py-setuptools', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-flye/package.py b/var/spack/repos/builtin/packages/py-flye/package.py index db3804e11d9..090d36cdb9e 100644 --- a/var/spack/repos/builtin/packages/py-flye/package.py +++ b/var/spack/repos/builtin/packages/py-flye/package.py @@ -26,7 +26,7 @@ class PyFlye(PythonPackage): msg = 'C++ compiler with C++11 support required' conflicts('%gcc@:4.7', msg=msg) conflicts('%clang@:3.2', msg=msg) - # Requires Apple Clang 5.0+ but no way to specify that right now + conflicts('apple-clang@:4.9', msg=msg) def setup_build_environment(self, env): if self.spec.target.family == 'aarch64': diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py index 52c9beae434..d73d3f7576e 100644 --- a/var/spack/repos/builtin/packages/py-scikit-learn/package.py +++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py @@ -69,9 +69,7 @@ class PyScikitLearn(PythonPackage): depends_on('py-pytest@3.3.0:', type='test') depends_on('py-pandas', type='test') depends_on('py-setuptools', type='build') - # Technically not correct, but currently no way to check if we - # are using Apple Clang or not. - depends_on('llvm-openmp', when='@0.21: %clang platform=darwin +openmp') + depends_on('llvm-openmp', when='@0.21: %apple-clang +openmp') # Release tarballs are already cythonized. If you wanted to build a release # version without OpenMP support, you would need to delete all .c files @@ -84,7 +82,7 @@ def setup_build_environment(self, env): if self.spec.satisfies('~openmp'): env.set('SKLEARN_NO_OPENMP', 'True') # https://scikit-learn.org/stable/developers/advanced_installation.html#mac-osx - elif self.spec.satisfies('@0.21: %clang platform=darwin +openmp'): + elif self.spec.satisfies('@0.21: %apple-clang +openmp'): env.append_flags( 'CPPFLAGS', self.compiler.openmp_flag) env.append_flags( diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py index c240144f11d..1ffeb81b23d 100644 --- a/var/spack/repos/builtin/packages/py-torch/package.py +++ b/var/spack/repos/builtin/packages/py-torch/package.py @@ -155,7 +155,7 @@ class PyTorch(PythonPackage, CudaPackage): depends_on('nccl', when='+nccl') depends_on('gloo', when='+gloo') depends_on('opencv', when='+opencv') - depends_on('llvm-openmp', when='%clang platform=darwin +openmp') + depends_on('llvm-openmp', when='%apple-clang +openmp') depends_on('ffmpeg', when='+ffmpeg') depends_on('leveldb', when='+leveldb') depends_on('lmdb', when='+lmdb') diff --git a/var/spack/repos/builtin/packages/qmcpack/package.py b/var/spack/repos/builtin/packages/qmcpack/package.py index 5307b3d2d5c..07724f63fea 100644 --- a/var/spack/repos/builtin/packages/qmcpack/package.py +++ b/var/spack/repos/builtin/packages/qmcpack/package.py @@ -102,6 +102,7 @@ class Qmcpack(CMakePackage, CudaPackage): conflicts('+afqmc', when='@:3.6.0', msg='AFQMC not recommended before v3.7') conflicts('+afqmc', when='~mpi', msg='AFQMC requires building with +mpi') conflicts('+afqmc', when='%gcc@:6.0', msg='AFQMC code requires gcc@6.1 or greater') + conflicts('+afqmc', when='%apple-clang@:9.2', msg='AFQMC code requires clang 4.1 or greater') conflicts('+afqmc', when='%clang@:4.0', msg='AFQMC code requires clang 4.1 or greater') conflicts('+afqmc', when='%intel@:18', msg='AFQMC code requires intel19 or greater') diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index ba28042fa3d..7e016846ab5 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -286,7 +286,7 @@ def repl(match): "qmake/qmake.pri", "src/tools/bootstrap/bootstrap.pro" ] - if '%clang' in self.spec: + if '%clang' in self.spec or '%apple-clang' in self.spec: files_to_filter += [ "mkspecs/unsupported/macx-clang-libc++/qmake.conf", "mkspecs/common/clang.conf" diff --git a/var/spack/repos/builtin/packages/raxml/package.py b/var/spack/repos/builtin/packages/raxml/package.py index 08d99e6b16b..7eb477dfc60 100644 --- a/var/spack/repos/builtin/packages/raxml/package.py +++ b/var/spack/repos/builtin/packages/raxml/package.py @@ -25,6 +25,7 @@ class Raxml(Package): # Compiles with either GCC or ICC. conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%nag') conflicts('%pgi') diff --git a/var/spack/repos/builtin/packages/sgpp/package.py b/var/spack/repos/builtin/packages/sgpp/package.py index 7e5b907374e..728638500f8 100644 --- a/var/spack/repos/builtin/packages/sgpp/package.py +++ b/var/spack/repos/builtin/packages/sgpp/package.py @@ -97,6 +97,7 @@ class Sgpp(SConsPackage): # Compiler with C++11 support is required conflicts('%gcc@:4.8.4', msg='Compiler with c++11 support is required!') + conflicts('%apple-clang@:3.9', msg='Compiler with c++11 support is required!') conflicts('%clang@:3.2', msg='Compiler with c++11 support is required!') conflicts('%intel@:14', msg='Compiler with c++11 support is required!') # Solver python bindings are actually using the pde module at one point: diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 9192937393f..3d9ef4e16c3 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -57,6 +57,13 @@ def flag_handler(self, name, flags): @when('%clang@9:') def patch(self): + self.clang_9_patch() + + @when('%apple-clang@11.0.3:') + def patch(self): + self.clang_9_patch() + + def clang_9_patch(self): # Clang 9 and later include macro definitions in that conflict # with typedefs DOMAIN and RANGE used in Silo plugins. # It looks like the upstream fpzip repo has been fixed, but that change diff --git a/var/spack/repos/builtin/packages/snap-berkeley/package.py b/var/spack/repos/builtin/packages/snap-berkeley/package.py index 90d970e2770..de9928e0c39 100644 --- a/var/spack/repos/builtin/packages/snap-berkeley/package.py +++ b/var/spack/repos/builtin/packages/snap-berkeley/package.py @@ -22,6 +22,7 @@ class SnapBerkeley(MakefilePackage): conflicts('%gcc@6:') conflicts('%cce') + conflicts('%apple-clang') conflicts('%clang') conflicts('%intel') conflicts('%nag') diff --git a/var/spack/repos/builtin/packages/sqlitebrowser/package.py b/var/spack/repos/builtin/packages/sqlitebrowser/package.py index 6cd8eeb9b8b..b92d65348ed 100644 --- a/var/spack/repos/builtin/packages/sqlitebrowser/package.py +++ b/var/spack/repos/builtin/packages/sqlitebrowser/package.py @@ -18,6 +18,7 @@ class Sqlitebrowser(CMakePackage): msg = 'sqlitebrowser requires C++11 support' conflicts('%gcc@:4.8.0', msg=msg) + conflicts('%apple-clang@:3.9', msg=msg) conflicts('%clang@:3.2', msg=msg) conflicts('%intel@:12', msg=msg) conflicts('%xl@:13.0', msg=msg) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index eef38fefae9..703fbcf6709 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -115,7 +115,7 @@ def install(self, spec, prefix): # CFLAGS, but not all compilers use the same flags for these # optimizations if any([x in spec - for x in ('%clang', '%gcc', '%intel')]): + for x in ('%apple-clang', '%clang', '%gcc', '%intel')]): make_args += ['CFLAGS+=-fno-common -fexceptions'] elif '%pgi' in spec: make_args += ['CFLAGS+=--exceptions'] diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py index e374a4bbcad..50832e6e5af 100644 --- a/var/spack/repos/builtin/packages/sundials/package.py +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -214,7 +214,7 @@ def on_off(varstr): return 'ON' if varstr in self.spec else 'OFF' fortran_flag = self.compiler.f77_pic_flag - if (spec.satisfies('%clang platform=darwin')) and ('+fcmix' in spec): + if (spec.satisfies('%apple-clang')) and ('+fcmix' in spec): f77 = Executable(self.compiler.f77) libgfortran = LibraryList(f77('--print-file-name', 'libgfortran.a', output=str)) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 16f916849c3..7896d809f2f 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -683,7 +683,9 @@ def define_tpl_enable(cmake_var, spec_var=None): # Fortran lib (assumes clang is built with gfortran!) if '+fortran' in spec and ( - spec.satisfies('%gcc') or spec.satisfies('%clang')): + spec.satisfies('%gcc') or spec.satisfies('%clang') or + spec.satisfies('%apple-clang') + ): options.append(define('Trilinos_ENABLE_Fortran', True)) if '+mpi' in spec: libgfortran = os.path.dirname(os.popen( diff --git a/var/spack/repos/builtin/packages/unblur/package.py b/var/spack/repos/builtin/packages/unblur/package.py index 428f44c08e9..7366ad836a5 100644 --- a/var/spack/repos/builtin/packages/unblur/package.py +++ b/var/spack/repos/builtin/packages/unblur/package.py @@ -27,6 +27,7 @@ class Unblur(AutotoolsPackage): # Requires Intel Fortran compiler conflicts('%gcc') conflicts('%pgi') + conflicts('%apple-clang') conflicts('%clang') conflicts('%cce') conflicts('%xl') diff --git a/var/spack/repos/builtin/packages/valgrind/package.py b/var/spack/repos/builtin/packages/valgrind/package.py index 4555b0b886f..20d8791f4ec 100644 --- a/var/spack/repos/builtin/packages/valgrind/package.py +++ b/var/spack/repos/builtin/packages/valgrind/package.py @@ -40,7 +40,7 @@ class Valgrind(AutotoolsPackage, SourcewarePackage): variant('ubsan', default=sys.platform != 'darwin', description='Activates ubsan support for valgrind') - conflicts('+ubsan', when='platform=darwin %clang', + conflicts('+ubsan', when='%apple-clang', msg=""" Cannot build libubsan with clang on macOS. Otherwise with (Apple's) clang there is a linker error: diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py index 0df4c4a5fca..2a334cb1fbc 100644 --- a/var/spack/repos/builtin/packages/vtk/package.py +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -285,9 +285,7 @@ def cmake_args(self): # string. This fix was recommended on the VTK mailing list # in March 2014 (see # https://public.kitware.com/pipermail/vtkusers/2014-March/083368.html) - if (self.spec.satisfies('%clang') and - self.compiler.is_apple and - self.compiler.version >= Version('5.1.0')): + if self.spec.satisfies('%apple-clang@5.1.0:'): cmake_args.extend(['-DVTK_REQUIRED_OBJCXX_FLAGS=']) # A bug in tao pegtl causes build failures with intel compilers diff --git a/var/spack/repos/builtin/packages/warpx/package.py b/var/spack/repos/builtin/packages/warpx/package.py index dc19f916fec..c564528b407 100644 --- a/var/spack/repos/builtin/packages/warpx/package.py +++ b/var/spack/repos/builtin/packages/warpx/package.py @@ -68,7 +68,8 @@ class Warpx(MakefilePackage): def edit(self, spec, prefix): comp = 'gcc' - vendors = {'%gcc': 'gcc', '%intel': 'intel', '%clang': 'llvm'} + vendors = {'%gcc': 'gcc', '%intel': 'intel', + '%apple-clang': 'llvm', '%clang': 'llvm'} for key, value in vendors.items(): if self.spec.satisfies(key): comp = value diff --git a/var/spack/repos/builtin/packages/xios/package.py b/var/spack/repos/builtin/packages/xios/package.py index 54eca9fead0..2b5c6f9dae0 100644 --- a/var/spack/repos/builtin/packages/xios/package.py +++ b/var/spack/repos/builtin/packages/xios/package.py @@ -31,6 +31,7 @@ class Xios(Package): # Workaround bug #17782 in llvm, where reading a double # followed by a character is broken (e.g. duration '1d'): + patch('llvm_bug_17782.patch', when='@1.1: %apple-clang') patch('llvm_bug_17782.patch', when='@1.1: %clang') depends_on('netcdf-c+mpi') @@ -45,6 +46,13 @@ class Xios(Package): @when('%clang') def patch(self): + self.patch_llvm() + + @when('%apple-clang') + def patch(self): + self.patch_llvm() + + def patch_llvm(self): """Fix type references that are ambiguous for clang.""" for dirpath, dirnames, filenames in os.walk('src'): for filename in filenames: @@ -99,12 +107,13 @@ def xios_fcm(self): param['BOOST_LIB_DIR'] = spec['boost'].prefix.lib param['BLITZ_INC_DIR'] = spec['blitz'].prefix.include param['BLITZ_LIB_DIR'] = spec['blitz'].prefix.lib - if spec.satisfies('%clang platform=darwin'): + if spec.satisfies('%apple-clang'): param['LIBCXX'] = '-lc++' else: param['LIBCXX'] = '-lstdc++' - if any(map(spec.satisfies, ('%gcc', '%intel', '%clang'))): + if any(map(spec.satisfies, + ('%gcc', '%intel', '%apple-clang', '%clang'))): text = r""" %CCOMPILER {MPICXX} %FCOMPILER {MPIFC} diff --git a/var/spack/repos/builtin/packages/xsdktrilinos/package.py b/var/spack/repos/builtin/packages/xsdktrilinos/package.py index b679ee72773..effb4404b0f 100644 --- a/var/spack/repos/builtin/packages/xsdktrilinos/package.py +++ b/var/spack/repos/builtin/packages/xsdktrilinos/package.py @@ -65,7 +65,9 @@ def cmake_args(self): ]) # Fortran lib - if spec.satisfies('%gcc') or spec.satisfies('%clang'): + if (spec.satisfies('%gcc') or + spec.satisfies('%clang') or + spec.satisfies('%apple-clang')): libgfortran = os.path.dirname(os.popen( '%s --print-file-name libgfortran.a' % join_path(mpi_bin, 'mpif90')).read()) diff --git a/var/spack/repos/builtin/packages/yaml-cpp/package.py b/var/spack/repos/builtin/packages/yaml-cpp/package.py index 11f1f9fbefb..a83471f23a0 100644 --- a/var/spack/repos/builtin/packages/yaml-cpp/package.py +++ b/var/spack/repos/builtin/packages/yaml-cpp/package.py @@ -35,9 +35,7 @@ class YamlCpp(CMakePackage): conflicts('%gcc@:4.7', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") conflicts('%clang@:3.3.0', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") - # currently we can't check for apple-clang's version - # conflicts('%clang@:4.0.0-apple', when='@0.6.0:', - # msg="versions 0.6.0: require c++11 support") + conflicts('%apple-clang@:4.0.0', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") conflicts('%intel@:11.1', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") conflicts('%xl@:13.1', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") conflicts('%xl_r@:13.1', when='@0.6.0:', msg="versions 0.6.0: require c++11 support") From e04c89f0867ba275bf7926a42f175f09ac6cd9dc Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 25 Jun 2020 12:27:48 -0400 Subject: [PATCH 25/38] mpich: Allow building with external hwloc library (#15305) This is in principle supported already, but requires a patch to avoid build errors. Closes #15302 Co-authored-by: Michael Kuhn --- .../repos/builtin/packages/mpich/package.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index d862f125224..a4851244766 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -31,6 +31,7 @@ class Mpich(AutotoolsPackage): version('3.1', sha256='fcf96dbddb504a64d33833dc455be3dda1e71c7b3df411dfcf9df066d7c32c39') version('3.0.4', sha256='cf638c85660300af48b6f776e5ecd35b5378d5905ec5d34c3da7a27da0acf0b3') + variant('hwloc', default=True, description='Use external hwloc package') variant('hydra', default=True, description='Build the hydra process manager') variant('romio', default=True, description='Enable ROMIO MPI I/O implementation') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') @@ -75,6 +76,13 @@ class Mpich(AutotoolsPackage): 'mpicc', 'mpicxx', 'mpif77', 'mpif90', 'mpifort', relative_root='bin' ) + # Fix using an external hwloc + # See https://github.com/pmodels/mpich/issues/4038 + # and https://github.com/pmodels/mpich/pull/3540 + patch('https://github.com/pmodels/mpich/commit/8a851b317ee57366cd15f4f28842063d8eff4483.patch', + sha256='eb982de3366d48cbc55eb5e0df43373a45d9f51df208abf0835a72dc6c0b4774', + when='@3.3 +hwloc') + # fix MPI_Barrier segmentation fault # see https://lists.mpich.org/pipermail/discuss/2016-May/004764.html # and https://lists.mpich.org/pipermail/discuss/2016-June/004768.html @@ -110,6 +118,8 @@ class Mpich(AutotoolsPackage): depends_on('findutils', type='build') depends_on('pkgconfig', type='build') + depends_on('hwloc@2.0.0:', when='@3.3: +hwloc') + depends_on('libfabric', when='netmod=ofi') # The ch3 ofi netmod results in crashes with libfabric 1.7 # See https://github.com/pmodels/mpich/issues/3665 @@ -137,6 +147,12 @@ class Mpich(AutotoolsPackage): depends_on("m4", when="@develop", type=("build")), depends_on("autoconf@2.67:", when='@develop', type=("build")) + # building with "+hwloc' also requires regenerating autotools files + depends_on('automake@1.15:', when='@3.3 +hwloc', type="build") + depends_on('libtool@2.4.4:', when='@3.3 +hwloc', type="build") + depends_on("m4", when="@3.3 +hwloc", type="build"), + depends_on("autoconf@2.67:", when='@3.3 +hwloc', type="build") + conflicts('device=ch4', when='@:3.2') conflicts('netmod=ofi', when='@:3.1.4') conflicts('netmod=ucx', when='device=ch3') @@ -208,7 +224,8 @@ def setup_dependent_package(self, module, dependent_spec): def autoreconf(self, spec, prefix): """Not needed usually, configure should be already there""" # If configure exists nothing needs to be done - if os.path.exists(self.configure_abs_path): + if (os.path.exists(self.configure_abs_path) and + not spec.satisfies('@3.3 +hwloc')): return # Else bootstrap with autotools bash = which('bash') @@ -229,6 +246,8 @@ def configure_args(self): config_args = [ '--disable-silent-rules', '--enable-shared', + '--with-hwloc-prefix={0}'.format( + spec['hwloc'].prefix if '^hwloc' in spec else 'embedded'), '--with-pm={0}'.format('hydra' if '+hydra' in spec else 'no'), '--{0}-romio'.format('enable' if '+romio' in spec else 'disable'), '--{0}-ibverbs'.format('with' if '+verbs' in spec else 'without'), From f936e3a1dbc43629e01a3a7e2cc1682419cb381b Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 25 Jun 2020 12:13:26 -0500 Subject: [PATCH 26/38] environment spec lists: improve ability to query architecture in when clauses (#17056) --- lib/spack/docs/environments.rst | 10 +++++++--- lib/spack/spack/environment.py | 6 ++++-- lib/spack/spack/test/cmd/env.py | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index b80e36b4b2b..cec7224a868 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -647,7 +647,7 @@ named list ``compilers`` is ``['%gcc', '%clang', '%intel']`` on spack: definitions: - compilers: ['%gcc', '%clang'] - - when: target == 'x86_64' + - when: arch.satisfies('x86_64:') compilers: ['%intel'] .. note:: @@ -666,8 +666,12 @@ The valid variables for a ``when`` clause are: #. ``target``. The target string of the default Spack architecture on the system. -#. ``architecture`` or ``arch``. The full string of the - default Spack architecture on the system. +#. ``architecture`` or ``arch``. A Spack spec satisfying the default Spack + architecture on the system. This supports querying via the ``satisfies`` + method, as shown above. + +#. ``arch_str``. The architecture string of the default Spack architecture + on the system. #. ``re``. The standard regex module in Python. diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index bc9730e4908..8eab70b544d 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -409,12 +409,14 @@ def _eval_conditional(string): """Evaluate conditional definitions using restricted variable scope.""" arch = architecture.Arch( architecture.platform(), 'default_os', 'default_target') + arch_spec = spack.spec.Spec('arch=%s' % arch) valid_variables = { 'target': str(arch.target), 'os': str(arch.os), 'platform': str(arch.platform), - 'arch': str(arch), - 'architecture': str(arch), + 'arch': arch_spec, + 'architecture': arch_spec, + 'arch_str': str(arch), 're': re, 'env': os.environ, 'hostname': socket.gethostname() diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 57037387318..46977d8eec3 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -1429,6 +1429,29 @@ def test_stack_definition_conditional_with_variable(tmpdir): assert Spec('callpath') in test.user_specs +def test_stack_definition_conditional_with_satisfaction(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + when: arch.satisfies('platform=foo') # will be "test" when testing + - packages: [callpath] + when: arch.satisfies('platform=test') + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') not in test.user_specs + assert Spec('mpileaks') not in test.user_specs + assert Spec('callpath') in test.user_specs + + def test_stack_definition_complex_conditional(tmpdir): filename = str(tmpdir.join('spack.yaml')) with open(filename, 'w') as f: From 096bd69a949225590feeaeaf16545d673c4cee6d Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 25 Jun 2020 12:34:09 -0500 Subject: [PATCH 27/38] prevent multiple version sigils in the same spec (#17246) * prevent multiple version sigils in the same spec * fix packages with malformed versions --- lib/spack/spack/spec.py | 35 ++++++++++++++----- lib/spack/spack/test/spec_syntax.py | 15 ++++++++ lib/spack/spack/version.py | 3 ++ .../packages/netlib-scalapack/package.py | 6 ++-- .../builtin/packages/protobuf/package.py | 4 +-- .../builtin/packages/py-oauthlib/package.py | 2 +- .../repos/builtin/packages/xeus/package.py | 2 +- 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b23142e101d..175d160855e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -511,8 +511,17 @@ def __init__(self, *args): raise TypeError( "__init__ takes 1 or 2 arguments. (%d given)" % nargs) - def _add_version(self, version): - self.versions.add(version) + def _add_versions(self, version_list): + # If it already has a non-trivial version list, this is an error + if self.versions and self.versions != vn.VersionList(':'): + # Note: This may be impossible to reach by the current parser + # Keeping it in case the implementation changes. + raise MultipleVersionError( + 'A spec cannot contain multiple version signifiers.' + ' Use a version list instead.') + self.versions = vn.VersionList() + for version in version_list: + self.versions.add(version) def _autospec(self, compiler_spec_like): if isinstance(compiler_spec_like, CompilerSpec): @@ -1050,9 +1059,16 @@ def dependents_dict(self, deptype='all'): # # Private routines here are called by the parser when building a spec. # - def _add_version(self, version): + def _add_versions(self, version_list): """Called by the parser to add an allowable version.""" - self.versions.add(version) + # If it already has a non-trivial version list, this is an error + if self.versions and self.versions != vn.VersionList(':'): + raise MultipleVersionError( + 'A spec cannot contain multiple version signifiers.' + ' Use a version list instead.') + self.versions = vn.VersionList() + for version in version_list: + self.versions.add(version) def _add_flag(self, name, value): """Called by the parser to add a known flag. @@ -4157,9 +4173,7 @@ def spec(self, name): while self.next: if self.accept(AT): vlist = self.version_list() - spec.versions = vn.VersionList() - for version in vlist: - spec._add_version(version) + spec._add_versions(vlist) elif self.accept(ON): name = self.variant() @@ -4251,8 +4265,7 @@ def compiler(self): compiler.versions = vn.VersionList() if self.accept(AT): vlist = self.version_list() - for version in vlist: - compiler._add_version(version) + compiler._add_versions(vlist) else: compiler.versions = vn.VersionList(':') return compiler @@ -4325,6 +4338,10 @@ class DuplicateDependencyError(spack.error.SpecError): """Raised when the same dependency occurs in a spec twice.""" +class MultipleVersionError(spack.error.SpecError): + """Raised when version constraints occur in a spec twice.""" + + class DuplicateCompilerSpecError(spack.error.SpecError): """Raised when the same compiler occurs in a spec twice.""" diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index c9a2100b09e..52a842912ea 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -20,6 +20,7 @@ from spack.spec import DuplicateArchitectureError from spack.spec import DuplicateDependencyError, DuplicateCompilerSpecError from spack.spec import SpecFilenameError, NoSuchSpecFileError +from spack.spec import MultipleVersionError from spack.variant import DuplicateVariantError @@ -149,6 +150,9 @@ def test_simple_dependence(self): self.check_parse("openmpi ^hwloc ^libunwind", "openmpi^hwloc^libunwind") + def test_version_after_compiler(self): + self.check_parse('foo@2.0%bar@1.0', 'foo %bar@1.0 @2.0') + def test_dependencies_with_versions(self): self.check_parse("openmpi ^hwloc@1.2e6") self.check_parse("openmpi ^hwloc@1.2e6:") @@ -432,6 +436,17 @@ def test_duplicate_variant(self): ] self._check_raises(DuplicateVariantError, duplicates) + def test_multiple_versions(self): + multiples = [ + 'x@1.2@2.3', + 'x@1.2:2.3@1.4', + 'x@1.2@2.3:2.4', + 'x@1.2@2.3,2.4', + 'x@1.2 +foo~bar @2.3', + 'x@1.2%y@1.2@2.3:2.4', + ] + self._check_raises(MultipleVersionError, multiples) + def test_duplicate_dependency(self): self._check_raises(DuplicateDependencyError, ["x ^y ^y"]) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index bb7a79772e4..0c7399c3fb5 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -782,6 +782,9 @@ def __reversed__(self): def __len__(self): return len(self.versions) + def __bool__(self): + return bool(self.versions) + @coerced def __eq__(self, other): return other is not None and self.versions == other.versions diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 75993c47010..26fe53190a6 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -41,11 +41,11 @@ class NetlibScalapack(CMakePackage): depends_on('cmake', when='@2.0.0:', type='build') # See: https://github.com/Reference-ScaLAPACK/scalapack/issues/9 - patch("cmake_fortran_mangle.patch", when='@2.0.2:@2.0.99') + patch("cmake_fortran_mangle.patch", when='@2.0.2:2.0.99') # See: https://github.com/Reference-ScaLAPACK/scalapack/pull/10 - patch("mpi2-compatibility.patch", when='@2.0.2:@2.0.99') + patch("mpi2-compatibility.patch", when='@2.0.2:2.0.99') # See: https://github.com/Reference-ScaLAPACK/scalapack/pull/16 - patch("int_overflow.patch", when='@2.0.0:@2.1.0') + patch("int_overflow.patch", when='@2.0.0:2.1.0') @property def libs(self): diff --git a/var/spack/repos/builtin/packages/protobuf/package.py b/var/spack/repos/builtin/packages/protobuf/package.py index 82d2d05d75b..1cbd94b3144 100644 --- a/var/spack/repos/builtin/packages/protobuf/package.py +++ b/var/spack/repos/builtin/packages/protobuf/package.py @@ -56,10 +56,10 @@ class Protobuf(Package): # first fixed in 3.4.0: https://github.com/google/protobuf/pull/3406 patch('pkgconfig.patch', when='@3.0.2:3.3.2') - patch('intel-v1.patch', when='@3.2:@3.6 %intel') + patch('intel-v1.patch', when='@3.2:3.6 %intel') # See https://github.com/protocolbuffers/protobuf/pull/7197 - patch('intel-v2.patch', when='@3.7:@3.11.4 %intel') + patch('intel-v2.patch', when='@3.7:3.11.4 %intel') patch('protoc2.5.0_aarch64.patch', sha256='7b44fcdb794f421174d619f83584e00a36012a16da09079e2fad9c12f7337451', when='@2.5.0 target=aarch64:') diff --git a/var/spack/repos/builtin/packages/py-oauthlib/package.py b/var/spack/repos/builtin/packages/py-oauthlib/package.py index 163db071ad7..93fbb9aaccd 100644 --- a/var/spack/repos/builtin/packages/py-oauthlib/package.py +++ b/var/spack/repos/builtin/packages/py-oauthlib/package.py @@ -32,6 +32,6 @@ class PyOauthlib(PythonPackage): depends_on('py-pytest-cov@2.6:', type='test') depends_on('py-nose', type='test', when='@2.0.2') - depends_on('py-unittest2', type='test', when='^python@2 @2.0.2') + depends_on('py-unittest2', type='test', when='^python@2.0.2') depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/xeus/package.py b/var/spack/repos/builtin/packages/xeus/package.py index 629bbb76a8c..756a130aa48 100644 --- a/var/spack/repos/builtin/packages/xeus/package.py +++ b/var/spack/repos/builtin/packages/xeus/package.py @@ -27,7 +27,7 @@ class Xeus(CMakePackage): depends_on('cppzmq@4.3.0:') depends_on('cryptopp@7.0.0:') depends_on('xtl@0.4.0:') - depends_on('nlohmann-json@3.2.0', when='@develop@0.15.0:') + depends_on('nlohmann-json@3.2.0', when='@develop,0.15.0:') depends_on('nlohmann-json@3.1.1', when='@0.14.1') depends_on('libuuid') From 7c54aa2eb088398d626b8ce4354623d62f3a4638 Mon Sep 17 00:00:00 2001 From: Omar Padron Date: Thu, 25 Jun 2020 14:27:20 -0400 Subject: [PATCH 28/38] add workaround for gitlab ci needs limit (#17219) * add workaround for gitlab ci needs limit * fix style/address review comments * convert filter obj to list * update command completion * remove dict comprehension * add workaround tests * fix sorting issue between disparate types * add indeces to format --- lib/spack/spack/ci.py | 7 +- lib/spack/spack/ci_needs_workaround.py | 47 +++++++ lib/spack/spack/ci_optimization.py | 8 +- lib/spack/spack/cmd/ci.py | 10 +- lib/spack/spack/test/ci.py | 170 +++++++++++++++++++++++++ share/spack/spack-completion.bash | 2 +- 6 files changed, 236 insertions(+), 8 deletions(-) create mode 100644 lib/spack/spack/ci_needs_workaround.py diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 230f25ccee2..580dbf29d2b 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -450,7 +450,7 @@ def format_job_needs(phase_name, strip_compilers, dep_jobs, def generate_gitlab_ci_yaml(env, print_summary, output_file, custom_spack_repo=None, custom_spack_ref=None, - run_optimizer=False): + run_optimizer=False, use_dependencies=False): # FIXME: What's the difference between one that opens with 'spack' # and one that opens with 'env'? This will only handle the former. with spack.concretize.disable_compiler_existence_check(): @@ -794,6 +794,11 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, import spack.ci_optimization as ci_opt sorted_output = ci_opt.optimizer(sorted_output) + # TODO(opadron): remove this or refactor + if use_dependencies: + import spack.ci_needs_workaround as cinw + sorted_output = cinw.needs_to_dependencies(sorted_output) + with open(output_file, 'w') as outf: outf.write(syaml.dump_config(sorted_output, default_flow_style=True)) diff --git a/lib/spack/spack/ci_needs_workaround.py b/lib/spack/spack/ci_needs_workaround.py new file mode 100644 index 00000000000..ba9f7c62aff --- /dev/null +++ b/lib/spack/spack/ci_needs_workaround.py @@ -0,0 +1,47 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import collections + +try: + # dynamically import to keep vermin from complaining + collections_abc = __import__('collections.abc') +except ImportError: + collections_abc = collections + + +get_job_name = lambda needs_entry: ( + needs_entry.get('job') if ( + isinstance(needs_entry, collections_abc.Mapping) and + needs_entry.get('artifacts', True)) + + else + + needs_entry if isinstance(needs_entry, str) + + else None) + + +def convert_job(job_entry): + if not isinstance(job_entry, collections_abc.Mapping): + return job_entry + + needs = job_entry.get('needs') + if needs is None: + return job_entry + + new_job = {} + new_job.update(job_entry) + del new_job['needs'] + + new_job['dependencies'] = list(filter( + (lambda x: x is not None), + (get_job_name(needs_entry) for needs_entry in needs))) + + return new_job + + +def needs_to_dependencies(yaml): + return dict((k, convert_job(v)) for k, v in yaml.items()) diff --git a/lib/spack/spack/ci_optimization.py b/lib/spack/spack/ci_optimization.py index 693802d06d5..d938a681e32 100644 --- a/lib/spack/spack/ci_optimization.py +++ b/lib/spack/spack/ci_optimization.py @@ -190,10 +190,10 @@ def print_delta(name, old, new, applied=None): applied = (new <= old) print('\n'.join(( - '{} {}:', - ' before: {: 10d}', - ' after : {: 10d}', - ' delta : {:+10d} ({:=+3d}.{}%)', + '{0} {1}:', + ' before: {2: 10d}', + ' after : {3: 10d}', + ' delta : {4:+10d} ({5:=+3d}.{6}%)', )).format( name, ('+' if applied else 'x'), diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 3e57c6656af..d42a3e7ae6d 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -55,10 +55,14 @@ def setup_parser(subparser): "should be checked out as a step in each generated job. " + "This argument is ignored if no --spack-repo is provided.") generate.add_argument( - '--optimize', action='store_true', + '--optimize', action='store_true', default=False, help="(Experimental) run the generated document through a series of " "optimization passes designed to reduce the size of the " "generated file.") + generate.add_argument( + '--dependencies', action='store_true', default=False, + help="(Experimental) disable DAG scheduling; use " + ' "plain" dependencies.') generate.set_defaults(func=ci_generate) # Check a spec against mirror. Rebuild, create buildcache and push to @@ -81,6 +85,7 @@ def ci_generate(args): spack_repo = args.spack_repo spack_ref = args.spack_ref run_optimizer = args.optimize + use_dependencies = args.dependencies if not output_file: gen_ci_dir = os.getcwd() @@ -93,7 +98,8 @@ def ci_generate(args): # Generate the jobs spack_ci.generate_gitlab_ci_yaml( env, True, output_file, spack_repo, spack_ref, - run_optimizer=run_optimizer) + run_optimizer=run_optimizer, + use_dependencies=use_dependencies) if copy_yaml_to: copy_to_dir = os.path.dirname(copy_yaml_to) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index a153437ffab..9b3c48654a2 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -15,6 +15,17 @@ import spack.util.web as web_util import spack.util.gpg +import spack.ci_optimization as ci_opt +import spack.ci_needs_workaround as cinw +import spack.util.spack_yaml as syaml +import itertools as it +import collections +try: + # dynamically import to keep vermin from complaining + collections_abc = __import__('collections.abc') +except ImportError: + collections_abc = collections + @pytest.fixture def tmp_scope(): @@ -166,3 +177,162 @@ def test_read_write_cdash_ids(config, tmp_scope, tmpdir, mock_packages): read_cdashid = ci.read_cdashid_from_mirror(mock_spec, mirror_url) assert(str(read_cdashid) == orig_cdashid) + + +def test_ci_workarounds(): + fake_root_spec = 'x' * 544 + fake_spack_ref = 'x' * 40 + + common_variables = { + 'SPACK_COMPILER_ACTION': 'NONE', + 'SPACK_IS_PR_PIPELINE': 'False', + } + + common_script = ['spack ci rebuild'] + + common_before_script = [ + 'git clone "https://github.com/spack/spack"', + ' && '.join(( + 'pushd ./spack', + 'git checkout "{ref}"'.format(ref=fake_spack_ref), + 'popd')), + '. "./spack/share/spack/setup-env.sh"' + ] + + def make_build_job(name, deps, stage, use_artifact_buildcache, optimize, + use_dependencies): + variables = common_variables.copy() + variables['SPACK_JOB_SPEC_PKG_NAME'] = name + + result = { + 'stage': stage, + 'tags': ['tag-0', 'tag-1'], + 'artifacts': { + 'paths': [ + 'jobs_scratch_dir', + 'cdash_report', + name + '.spec.yaml', + name + '.cdashid', + name + ], + 'when': 'always' + }, + 'retry': {'max': 2, 'when': ['always']}, + 'after_script': ['rm -rf "./spack"'], + 'image': {'name': 'spack/centos7', 'entrypoint': ['']}, + } + + if optimize: + result['extends'] = ['.c0', '.c1', '.c2'] + else: + variables['SPACK_ROOT_SPEC'] = fake_root_spec + result['script'] = common_script + result['before_script'] = common_before_script + + result['variables'] = variables + + if use_dependencies: + result['dependencies'] = ( + list(deps) if use_artifact_buildcache + else []) + else: + result['needs'] = [ + {'job': dep, 'artifacts': use_artifact_buildcache} + for dep in deps] + + return {name: result} + + def make_rebuild_index_job( + use_artifact_buildcache, optimize, use_dependencies): + + result = { + 'stage': 'stage-rebuild-index', + 'script': 'spack buildcache update-index -d s3://mirror', + 'tags': ['tag-0', 'tag-1'], + 'image': {'name': 'spack/centos7', 'entrypoint': ['']}, + 'after_script': ['rm -rf "./spack"'], + } + + if optimize: + result['extends'] = '.c1' + else: + result['before_script'] = common_before_script + + return {'rebuild-index': result} + + def make_factored_jobs(optimize): + return { + '.c0': {'script': common_script}, + '.c1': {'before_script': common_before_script}, + '.c2': {'variables': {'SPACK_ROOT_SPEC': fake_root_spec}} + } if optimize else {} + + def make_yaml_obj(use_artifact_buildcache, optimize, use_dependencies): + result = {} + + result.update(make_build_job( + 'pkg-a', [], 'stage-0', use_artifact_buildcache, optimize, + use_dependencies)) + + result.update(make_build_job( + 'pkg-b', ['pkg-a'], 'stage-1', use_artifact_buildcache, optimize, + use_dependencies)) + + result.update(make_build_job( + 'pkg-c', ['pkg-a', 'pkg-b'], 'stage-2', use_artifact_buildcache, + optimize, use_dependencies)) + + result.update(make_rebuild_index_job( + use_artifact_buildcache, optimize, use_dependencies)) + + result.update(make_factored_jobs(optimize)) + + return result + + def sort_yaml_obj(obj): + if isinstance(obj, collections_abc.Mapping): + result = syaml.syaml_dict() + for k in sorted(obj.keys(), key=str): + result[k] = sort_yaml_obj(obj[k]) + return result + + if (isinstance(obj, collections_abc.Sequence) and + not isinstance(obj, str)): + return syaml.syaml_list(sorted( + (sort_yaml_obj(x) for x in obj), key=str)) + + return obj + + # test every combination of: + # use artifact buildcache: true or false + # run optimization pass: true or false + # convert needs to dependencies: true or false + for use_ab in (False, True): + original = make_yaml_obj( + use_artifact_buildcache=use_ab, + optimize=False, + use_dependencies=False) + + for opt, deps in it.product(*(((False, True),) * 2)): + # neither optimizing nor converting needs->dependencies + if not (opt or deps): + # therefore, nothing to test + continue + + predicted = make_yaml_obj( + use_artifact_buildcache=use_ab, + optimize=opt, + use_dependencies=deps) + + actual = original.copy() + if opt: + actual = ci_opt.optimizer(actual) + if deps: + actual = cinw.needs_to_dependencies(actual) + + predicted = syaml.dump_config( + sort_yaml_obj(predicted), default_flow_style=True) + actual = syaml.dump_config( + sort_yaml_obj(actual), default_flow_style=True) + + assert(predicted == actual) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index ffd75964164..eff9e15b2c8 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -474,7 +474,7 @@ _spack_ci() { } _spack_ci_generate() { - SPACK_COMPREPLY="-h --help --output-file --copy-to --spack-repo --spack-ref --optimize" + SPACK_COMPREPLY="-h --help --output-file --copy-to --spack-repo --spack-ref --optimize --dependencies" } _spack_ci_rebuild() { From 9f3e542d30a94b7f23b1f55eb8958db4072190ea Mon Sep 17 00:00:00 2001 From: Andrey Prokopenko Date: Thu, 25 Jun 2020 15:20:31 -0400 Subject: [PATCH 29/38] Update ArborX for new Kokkos (#17213) * Update ArborX for new Kokkos * Set maintainer --- .../repos/builtin/packages/arborx/package.py | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/arborx/package.py b/var/spack/repos/builtin/packages/arborx/package.py index 7d10ef1fabb..5353b09de2f 100644 --- a/var/spack/repos/builtin/packages/arborx/package.py +++ b/var/spack/repos/builtin/packages/arborx/package.py @@ -9,42 +9,63 @@ class Arborx(CMakePackage): """ArborX is a performance-portable library for geometric search""" - homepage = "http://github.com/arborx/arborx" - url = "https://github.com/arborx/arborx/archive/v0.8-beta2.tar.gz" + homepage = "https://github.com/arborx/arborx" + url = "https://github.com/arborx/arborx/archive/v0.9-beta.tar.gz" git = "https://github.com/arborx/arborx.git" - version('master', branch='master') - version('0.8-beta2', sha256='e68733bc77fbb84313f3ff059f746fa79ab2ffe24a0a391126eefa47ec4fd2df') + maintainers = ['aprokop'] + + version('master', branch='master') + version('0.9-beta', sha256='b349b5708d1aa00e8c20c209ac75dc2d164ff9bf1b85adb5437346d194ba6c0d') + + # ArborX relies on Kokkos to provide devices, providing one-to-one matching + # variants. The only way to disable those devices is to make sure Kokkos + # does not provide them. + kokkos_backends = { + 'serial': (True, "enable Serial backend (default)"), + 'cuda': (False, "enable Cuda backend"), + 'openmp': (False, "enable OpenMP backend"), + 'hip': (False, "enable HIP backend") + } - variant('cuda', default=False, description='enable Cuda backend') - variant('openmp', default=False, description='enable OpenMP backend') - variant('serial', default=True, description='enable Serial backend (default)') variant('mpi', default=True, description='enable MPI') + for backend in kokkos_backends: + deflt, descr = kokkos_backends[backend] + variant(backend.lower(), default=deflt, description=descr) + variant('trilinos', default=False, description='use Kokkos from Trilinos') depends_on('cmake@3.12:', type='build') - depends_on('cuda', when='+cuda') depends_on('mpi', when='+mpi') - # ArborX relies on Kokkos to provide devices, thus having one-to-one match - # The only way to disable those devices is to make sure Kokkos does not - # provide them - depends_on('kokkos-legacy@2.7.00:+cuda+enable_lambda cxxstd=c++14', when='+cuda') - depends_on('kokkos-legacy@2.7.00:+openmp cxxstd=c++14', when='+openmp') - depends_on('kokkos-legacy@2.7.00:+serial cxxstd=c++14', when='+serial') + # Standalone Kokkos + depends_on('kokkos@3.1.00:', when='~trilinos') + for backend in kokkos_backends: + depends_on('kokkos+%s' % backend.lower(), when='~trilinos+%s' % + backend.lower()) + depends_on('kokkos+cuda_lambda', when='~trilinos+cuda') + + # Trilinos/Kokkos + # Notes: + # - there is no Trilinos release with Kokkos 3.1 yet + # - current version of Trilinos package does not allow disabling Serial + # - current version of Trilinos package does not allow enabling CUDA + depends_on('trilinos+kokkos@develop', when='+trilinos') + depends_on('trilinos+openmp', when='trilinos+openmp') + conflicts('~serial', when='+trilinos') + conflicts('+cuda', when='+trilinos') def cmake_args(self): spec = self.spec options = [ - '-DCMAKE_PREFIX_PATH=%s' % spec['kokkos-legacy'].prefix, - '-DARBORX_ENABLE_TESTS=OFF', - '-DARBORX_ENABLE_EXAMPLES=OFF', - '-DARBORX_ENABLE_BENCHMARKS=OFF', + '-DKokkos_ROOT=%s' % (spec['kokkos'].prefix if '~trilinos' in spec + else spec['trilinos'].prefix), '-DARBORX_ENABLE_MPI=%s' % ('ON' if '+mpi' in spec else 'OFF') ] if '+cuda' in spec: - nvcc_wrapper_path = spec['kokkos'].prefix.bin.nvcc_wrapper - options.append('-DCMAKE_CXX_COMPILER=%s' % nvcc_wrapper_path) + # Only Kokkos allows '+cuda' for now + options.append( + '-DCMAKE_CXX_COMPILER=%s' % spec["kokkos"].kokkos_cxx) return options From e8465ce81c004a28f6231c439d7feefc4cbfc8e7 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 25 Jun 2020 21:32:34 +0200 Subject: [PATCH 30/38] libtirpc: Add 1.2.6 (#17209) Earlier versions do not compile with gcc@10:. --- var/spack/repos/builtin/packages/libtirpc/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/libtirpc/package.py b/var/spack/repos/builtin/packages/libtirpc/package.py index f83c7499a09..f38e8fa8d60 100644 --- a/var/spack/repos/builtin/packages/libtirpc/package.py +++ b/var/spack/repos/builtin/packages/libtirpc/package.py @@ -13,12 +13,14 @@ class Libtirpc(AutotoolsPackage): homepage = "https://sourceforge.net/projects/libtirpc/" url = "https://sourceforge.net/projects/libtirpc/files/libtirpc/1.1.4/libtirpc-1.1.4.tar.bz2/download" + version('1.2.6', sha256='4278e9a5181d5af9cd7885322fdecebc444f9a3da87c526e7d47f7a12a37d1cc') version('1.1.4', sha256='2ca529f02292e10c158562295a1ffd95d2ce8af97820e3534fe1b0e3aec7561d') provides('rpc') # FIXME: build error on macOS # auth_none.c:81:9: error: unknown type name 'mutex_t' + conflicts('platform=darwin', msg='Does not build on macOS') def configure_args(self): return ['--disable-gssapi'] From d2b56dbce3f6a847a015bb002e7748fe214a154a Mon Sep 17 00:00:00 2001 From: manifest Date: Fri, 26 Jun 2020 05:03:26 +0930 Subject: [PATCH 31/38] add sra-toolkit@2.10.7 (#16947) * sra-tools * sra-tools style fix * sratoolkit version update --- var/spack/repos/builtin/packages/sratoolkit/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/sratoolkit/package.py b/var/spack/repos/builtin/packages/sratoolkit/package.py index c399d1b617a..d38de8eead5 100644 --- a/var/spack/repos/builtin/packages/sratoolkit/package.py +++ b/var/spack/repos/builtin/packages/sratoolkit/package.py @@ -14,6 +14,9 @@ class Sratoolkit(Package): homepage = "https://trace.ncbi.nlm.nih.gov/Traces/sra" url = "https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/2.8.2-1/sratoolkit.2.8.2-1-centos_linux64.tar.gz" + maintainers = ['robqiao'] + + version('2.10.7', sha256='b3f319974f0c7a318554d6383a13dd30f7d447533d92b6fd3bd057d3524e0140') version('2.9.6', sha256='faab687c822d0c02956f73f35e04875dde420ce9f602b88bbf3f2e8d79a17155') version('2.9.2', sha256='17dbe13aa1ed7955d31e1e76e8b62786e80a77e9ed9d396631162dc3ad8b716d') version('2.8.2-1', sha256='b053061aae7c6d00162fe0f514be4128a60365b4b2b5b36e7f4798b348b55cf5') From 740a9d88f9e873af2ba05c3ae1029c82e47b0d22 Mon Sep 17 00:00:00 2001 From: Ge Wenjun <39968398+wjge@users.noreply.github.com> Date: Thu, 25 Jun 2020 15:57:31 -0400 Subject: [PATCH 32/38] add new package: datatransferkit (#17158) * add new package: datatransferkit * fix style * remove the build type;add~dtk;'shared' variant;homepage * add maintainer --- .../packages/datatransferkit/package.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 var/spack/repos/builtin/packages/datatransferkit/package.py diff --git a/var/spack/repos/builtin/packages/datatransferkit/package.py b/var/spack/repos/builtin/packages/datatransferkit/package.py new file mode 100644 index 00000000000..959d5b50935 --- /dev/null +++ b/var/spack/repos/builtin/packages/datatransferkit/package.py @@ -0,0 +1,46 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Datatransferkit(CMakePackage): + """DataTransferKit is an open-source software library of + parallel solution transfer services for multiphysics simulations""" + + homepage = "https://datatransferkit.readthedoc.io" + url = "https://github.com/ORNL-CEES/DataTransferKit/archive/3.1-rc1.tar.gz" + git = "https://github.com/ORNL-CEES/DataTransferKit.git" + + maintainers = ['Rombur'] + + version('master', branch='master', submodules=True) + + variant('openmp', default=False, description='enable OpenMP backend') + variant('serial', default=True, description='enable Serial backend (default)') + variant('shared', default=True, + description='enable the build of shared lib') + + depends_on('cmake', type='build') + depends_on('trilinos@develop+intrepid2+shards~dtk', when='+serial') + depends_on('trilinos@develop+intrepid2+shards+openmp~dtk', when='+openmp') + + def cmake_args(self): + spec = self.spec + + options = [ + '-DBUILD_SHARED_LIBS:BOOL=%s' % ( + 'ON' if '+shared' in spec else 'OFF'), + '-DDataTransferKit_ENABLE_DataTransferKit=ON', + '-DDataTransferKit_ENABLE_TESTS=OFF', + '-DDataTransferKit_ENABLE_EXAMPLES=OFF', + '-DCMAKE_CXX_EXTENSIONS=OFF', + '-DCMAKE_CXX_STANDARD=14', + ] + + if '+openmp' in spec: + options.append('-DDataTransferKit_ENABLE_OpenMP=ON') + + return options From 49b0ac804b227f7964d02b156b569e7c31fcc6dc Mon Sep 17 00:00:00 2001 From: "Dr. Christian Tacke" <58549698+ChristianTackeGSI@users.noreply.github.com> Date: Thu, 25 Jun 2020 22:33:38 +0200 Subject: [PATCH 33/38] munge: Update homepage URL (#17257) --- var/spack/repos/builtin/packages/munge/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/munge/package.py b/var/spack/repos/builtin/packages/munge/package.py index 66d14ec5d72..2b47716aa2b 100644 --- a/var/spack/repos/builtin/packages/munge/package.py +++ b/var/spack/repos/builtin/packages/munge/package.py @@ -9,7 +9,7 @@ class Munge(AutotoolsPackage): """ MUNGE Uid 'N' Gid Emporium """ - homepage = "https://code.google.com/p/munge/" + homepage = "https://dun.github.io/munge/" url = "https://github.com/dun/munge/releases/download/munge-0.5.14/munge-0.5.14.tar.xz" maintainers = ['ChristianTackeGSI'] From e6bf0b01c43ba60a1e61cc77ebc37ccce6d51b05 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 25 Jun 2020 18:14:57 -0700 Subject: [PATCH 34/38] Ascent & Conduit: ~python default (#17230) * Ascent: ~python default Packages that build optional python bindings do not build them by default in Spack: https://spack.readthedocs.io/en/latest/packaging_guide.html#variant-names This reduces long dependency trees and build times, e.g. for apps just using C/C++/Fortran bindings of a library. * Conduit: ~python default Packages that build optional python bindings do not build them by default in Spack: https://spack.readthedocs.io/en/latest/packaging_guide.html#variant-names This reduces long dependency trees and build times, e.g. for apps just using C/C++/Fortran bindings of a library. --- var/spack/repos/builtin/packages/ascent/package.py | 4 ++-- var/spack/repos/builtin/packages/conduit/package.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/ascent/package.py b/var/spack/repos/builtin/packages/ascent/package.py index 242f17e8a2c..3b45c69694d 100644 --- a/var/spack/repos/builtin/packages/ascent/package.py +++ b/var/spack/repos/builtin/packages/ascent/package.py @@ -50,13 +50,13 @@ class Ascent(Package, CudaPackage): ########################################################################### variant("shared", default=True, description="Build Ascent as shared libs") - variant('test', default=True, description='Enable Ascent unit tests') + variant("test", default=True, description='Enable Ascent unit tests') variant("mpi", default=True, description="Build Ascent MPI Support") variant("serial", default=True, description="build serial (non-mpi) libraries") # variants for language support - variant("python", default=True, description="Build Ascent Python support") + variant("python", default=False, description="Build Ascent Python support") variant("fortran", default=True, description="Build Ascent Fortran support") # variants for runtime features diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py index 8996de7d9d4..0e2ae11f58a 100644 --- a/var/spack/repos/builtin/packages/conduit/package.py +++ b/var/spack/repos/builtin/packages/conduit/package.py @@ -54,10 +54,10 @@ class Conduit(Package): ########################################################################### variant("shared", default=True, description="Build Conduit as shared libs") - variant('test', default=True, description='Enable Conduit unit tests') + variant("test", default=True, description='Enable Conduit unit tests') # variants for python support - variant("python", default=True, description="Build Conduit Python support") + variant("python", default=False, description="Build Conduit Python support") variant("fortran", default=True, description="Build Conduit Fortran support") # variants for comm and i/o From cecd300693e654ee0bbd516a05c430475bbb76e0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 25 Jun 2020 20:15:21 -0500 Subject: [PATCH 35/38] py-azureml-sdk: add new package (and deps) (#17261) --- .../repos/builtin/packages/py-adal/package.py | 23 ++++++++ .../py-applicationinsights/package.py | 16 ++++++ .../packages/py-azure-common/package.py | 16 ++++++ .../builtin/packages/py-azure-core/package.py | 20 +++++++ .../packages/py-azure-graphrbac/package.py | 19 +++++++ .../packages/py-azure-identity/package.py | 24 +++++++++ .../py-azure-mgmt-authorization/package.py | 19 +++++++ .../package.py | 19 +++++++ .../py-azure-mgmt-keyvault/package.py | 19 +++++++ .../packages/py-azure-mgmt-network/package.py | 19 +++++++ .../packages/py-azure-mgmt-nspkg/package.py | 16 ++++++ .../py-azure-mgmt-resource/package.py | 19 +++++++ .../packages/py-azure-mgmt-storage/package.py | 19 +++++++ .../packages/py-azure-nspkg/package.py | 15 ++++++ .../py-azureml-automl-core/package.py | 24 +++++++++ .../packages/py-azureml-core/package.py | 52 +++++++++++++++++++ .../py-azureml-dataprep-native/package.py | 27 ++++++++++ .../packages/py-azureml-dataprep/package.py | 28 ++++++++++ .../py-azureml-pipeline-core/package.py | 22 ++++++++ .../py-azureml-pipeline-steps/package.py | 24 +++++++++ .../packages/py-azureml-pipeline/package.py | 24 +++++++++ .../packages/py-azureml-sdk/package.py | 34 ++++++++++++ .../packages/py-azureml-telemetry/package.py | 24 +++++++++ .../py-azureml-train-automl-client/package.py | 27 ++++++++++ .../packages/py-azureml-train-core/package.py | 26 ++++++++++ .../package.py | 25 +++++++++ .../packages/py-azureml-train/package.py | 24 +++++++++ .../builtin/packages/py-distro/package.py | 15 ++++++ .../builtin/packages/py-docker/package.py | 21 ++++++++ .../packages/py-dotnetcore2/package.py | 28 ++++++++++ .../builtin/packages/py-fusepy/package.py | 5 +- .../builtin/packages/py-jeepney/package.py | 15 ++++++ .../builtin/packages/py-jsonpickle/package.py | 18 +++++++ .../packages/py-msal-extensions/package.py | 23 ++++++++ .../repos/builtin/packages/py-msal/package.py | 20 +++++++ .../builtin/packages/py-msrest/package.py | 21 ++++++++ .../packages/py-msrestazure/package.py | 18 +++++++ .../packages/py-ndg-httpsclient/package.py | 19 +++++++ .../builtin/packages/py-pyjwt/package.py | 5 +- .../packages/py-ruamel-ordereddict/package.py | 19 +++++++ .../packages/py-ruamel-yaml-clib/package.py | 17 ++++++ .../packages/py-ruamel-yaml/package.py | 12 +++-- .../packages/py-secretstorage/package.py | 18 +++++++ .../packages/py-setuptools-scm/package.py | 9 +++- .../packages/py-websocket-client/package.py | 19 +++++++ 45 files changed, 917 insertions(+), 9 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-adal/package.py create mode 100644 var/spack/repos/builtin/packages/py-applicationinsights/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-common/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-core/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-graphrbac/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-identity/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-authorization/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-containerregistry/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-keyvault/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-network/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-nspkg/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-resource/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-mgmt-storage/package.py create mode 100644 var/spack/repos/builtin/packages/py-azure-nspkg/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-automl-core/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-core/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-dataprep-native/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-dataprep/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-pipeline-core/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-pipeline-steps/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-pipeline/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-sdk/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-telemetry/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-train-automl-client/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-train-core/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-train-restclients-hyperdrive/package.py create mode 100644 var/spack/repos/builtin/packages/py-azureml-train/package.py create mode 100644 var/spack/repos/builtin/packages/py-distro/package.py create mode 100644 var/spack/repos/builtin/packages/py-docker/package.py create mode 100644 var/spack/repos/builtin/packages/py-dotnetcore2/package.py create mode 100644 var/spack/repos/builtin/packages/py-jeepney/package.py create mode 100644 var/spack/repos/builtin/packages/py-jsonpickle/package.py create mode 100644 var/spack/repos/builtin/packages/py-msal-extensions/package.py create mode 100644 var/spack/repos/builtin/packages/py-msal/package.py create mode 100644 var/spack/repos/builtin/packages/py-msrest/package.py create mode 100644 var/spack/repos/builtin/packages/py-msrestazure/package.py create mode 100644 var/spack/repos/builtin/packages/py-ndg-httpsclient/package.py create mode 100644 var/spack/repos/builtin/packages/py-ruamel-ordereddict/package.py create mode 100644 var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py create mode 100644 var/spack/repos/builtin/packages/py-secretstorage/package.py create mode 100644 var/spack/repos/builtin/packages/py-websocket-client/package.py diff --git a/var/spack/repos/builtin/packages/py-adal/package.py b/var/spack/repos/builtin/packages/py-adal/package.py new file mode 100644 index 00000000000..b1fac54c0f3 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-adal/package.py @@ -0,0 +1,23 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAdal(PythonPackage): + """The ADAL for Python library makes it easy for python application to + authenticate to Azure Active Directory (AAD) in order to access AAD + protected web resources. + + DEPRECATED: replaced by MSAL Python.""" + + homepage = "https://github.com/AzureAD/azure-activedirectory-library-for-python" + url = "https://pypi.io/packages/source/a/adal/adal-1.2.4.tar.gz" + + version('1.2.4', sha256='7a15d22b1ee7ce1be92441199958748982feba6b7dec35fbf60f9b607bad1bc0') + + depends_on('py-setuptools', type='build') + depends_on('py-pyjwt@1.0.0:', type=('build', 'run')) + depends_on('py-requests@2.0.0:', type=('build', 'run')) + depends_on('py-python-dateutil@2.1.0:', type=('build', 'run')) + depends_on('py-cryptography@1.1.0:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-applicationinsights/package.py b/var/spack/repos/builtin/packages/py-applicationinsights/package.py new file mode 100644 index 00000000000..a14808ababa --- /dev/null +++ b/var/spack/repos/builtin/packages/py-applicationinsights/package.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyApplicationinsights(PythonPackage): + """This project extends the Application Insights API surface to support + Python.""" + + homepage = "https://github.com/Microsoft/ApplicationInsights-Python" + url = "https://pypi.io/packages/source/a/applicationinsights/applicationinsights-0.11.9.tar.gz" + + version('0.11.9', sha256='30a11aafacea34f8b160fbdc35254c9029c7e325267874e3c68f6bdbcd6ed2c3') + + depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-azure-common/package.py b/var/spack/repos/builtin/packages/py-azure-common/package.py new file mode 100644 index 00000000000..72410b447e0 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-common/package.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureCommon(PythonPackage): + """Microsoft Azure Client Library for Python (Common).""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-common/azure-common-1.1.25.zip" + + version('1.1.25', sha256='ce0f1013e6d0e9faebaf3188cc069f4892fc60a6ec552e3f817c1a2f92835054') + + depends_on('py-setuptools', type='build') + depends_on('py-azure-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-core/package.py b/var/spack/repos/builtin/packages/py-azure-core/package.py new file mode 100644 index 00000000000..cfe14b0dae9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-core/package.py @@ -0,0 +1,20 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureCore(PythonPackage): + """Microsoft Azure Core Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/core/azure-core" + url = "https://pypi.io/packages/source/a/azure-core/azure-core-1.6.0.zip" + + version('1.6.0', sha256='d10b74e783cff90d56360e61162afdd22276d62dc9467e657ae866449eae7648') + + depends_on('py-setuptools', type='build') + depends_on('py-requests@2.18.4:', type=('build', 'run')) + depends_on('py-six@1.6:', type=('build', 'run')) + depends_on('py-azure-nspkg', when='^python@:2', type=('build', 'run')) + depends_on('py-enum34@1.0.4:', when='^python@:3.3', type=('build', 'run')) + depends_on('py-typing', when='^python@:3.4', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-graphrbac/package.py b/var/spack/repos/builtin/packages/py-azure-graphrbac/package.py new file mode 100644 index 00000000000..60a87b0a929 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-graphrbac/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureGraphrbac(PythonPackage): + """Microsoft Azure Graph RBAC Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-graphrbac/azure-graphrbac-0.61.1.zip" + + version('0.61.1', sha256='53e98ae2ca7c19b349e9e9bb1b6a824aeae8dcfcbe17190d20fe69c0f185b2e2') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-identity/package.py b/var/spack/repos/builtin/packages/py-azure-identity/package.py new file mode 100644 index 00000000000..e3835d9cefe --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-identity/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureIdentity(PythonPackage): + """Microsoft Azure Identity Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/identity/azure-identity" + url = "https://pypi.io/packages/source/a/azure-identity/azure-identity-1.3.1.zip" + + version('1.3.1', sha256='5a59c36b4b05bdaec455c390feda71b6495fc828246593404351b9a41c2e877a') + version('1.2.0', sha256='b32acd1cdb6202bfe10d9a0858dc463d8960295da70ae18097eb3b85ab12cb91') + + depends_on('py-setuptools', type='build') + depends_on('py-azure-core@1.0.0:1.999', type=('build', 'run')) + depends_on('py-cryptography@2.1.4:', type=('build', 'run')) + depends_on('py-msal@1.0.0:1.999', type=('build', 'run')) + depends_on('py-msal-extensions@0.1.3:0.1.999', type=('build', 'run')) + depends_on('py-six@1.6:', type=('build', 'run')) + depends_on('py-azure-nspkg', when='^python@:2', type=('build', 'run')) + depends_on('py-mock', when='^python@:3.2', type=('build', 'run')) + depends_on('py-typing', when='^python@:3.4', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-authorization/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-authorization/package.py new file mode 100644 index 00000000000..acc9d25e661 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-authorization/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtAuthorization(PythonPackage): + """Microsoft Azure Authorization Management Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-authorization/azure-mgmt-authorization-0.60.0.zip" + + version('0.60.0', sha256='31e875a34ac2c5d6fefe77b4a8079a8b2bdbe9edb957e47e8b44222fb212d6a7') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-containerregistry/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-containerregistry/package.py new file mode 100644 index 00000000000..ac444980353 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-containerregistry/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtContainerregistry(PythonPackage): + """Microsoft Azure Container Registry Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-containerregistry/azure-mgmt-containerregistry-2.8.0.zip" + + version('2.8.0', sha256='b24be1050d54f3158e8be7f6ad677f0c8888dddefd09fb8391ebfc73d40173a4') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-keyvault/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-keyvault/package.py new file mode 100644 index 00000000000..5273aec0ffa --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-keyvault/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtKeyvault(PythonPackage): + """Microsoft Azure Key Vault Management Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-keyvault/azure-mgmt-keyvault-2.2.0.zip" + + version('2.2.0', sha256='1883e12eeb5819064dc52bf3a3ade05c791f4b66e4aeec948bda28df6ce2bce4') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-network/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-network/package.py new file mode 100644 index 00000000000..cbc0473e799 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-network/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtNetwork(PythonPackage): + """Microsoft Azure Network Management Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-network/azure-mgmt-network-10.2.0.zip" + + version('10.2.0', sha256='d50c74cdc1c9be6861ddef9adffd3b05afc5a5092baf0209eea30f4439cba2d9') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-nspkg/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-nspkg/package.py new file mode 100644 index 00000000000..4c4dda2d0d0 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-nspkg/package.py @@ -0,0 +1,16 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtNspkg(PythonPackage): + """Microsoft Azure Resource Management Namespace Package [Internal].""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-nspkg/azure-mgmt-nspkg-3.0.2.zip" + + version('3.0.2', sha256='8b2287f671529505b296005e6de9150b074344c2c7d1c805b3f053d081d58c52') + + depends_on('py-setuptools', type='build') + depends_on('py-azure-nspkg@3.0.0:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-resource/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-resource/package.py new file mode 100644 index 00000000000..f22f77fd46d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-resource/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtResource(PythonPackage): + """Microsoft Azure Resource Management Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-resource/azure-mgmt-resource-10.0.0.zip" + + version('10.0.0', sha256='bd9a3938f5423741329436d2da09693845c2fad96c35fadbd7c5ae5213208345') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-mgmt-storage/package.py b/var/spack/repos/builtin/packages/py-azure-mgmt-storage/package.py new file mode 100644 index 00000000000..c2dd7d68f5f --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-mgmt-storage/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureMgmtStorage(PythonPackage): + """Microsoft Azure Storage Management Client Library for Python.""" + + homepage = "https://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-mgmt-storage/azure-mgmt-storage-11.0.0.zip" + + version('11.0.0', sha256='f9791c2a84eee0a55bbf757632a2a4d1e102db958e75422d5e0e7306041129b8') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.5.0:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.32:1.999', type=('build', 'run')) + depends_on('py-azure-common@1.1:1.999', type=('build', 'run')) + depends_on('py-azure-mgmt-nspkg', when='^python@:2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-azure-nspkg/package.py b/var/spack/repos/builtin/packages/py-azure-nspkg/package.py new file mode 100644 index 00000000000..8e1b4685656 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azure-nspkg/package.py @@ -0,0 +1,15 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzureNspkg(PythonPackage): + """Microsoft Azure Namespace Package [Internal].""" + + homepage = "hhttps://github.com/Azure/azure-sdk-for-python" + url = "https://pypi.io/packages/source/a/azure-nspkg/azure-nspkg-3.0.2.zip" + + version('3.0.2', sha256='e7d3cea6af63e667d87ba1ca4f8cd7cb4dfca678e4c55fc1cedb320760e39dd0') + + depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-azureml-automl-core/package.py b/var/spack/repos/builtin/packages/py-azureml-automl-core/package.py new file mode 100644 index 00000000000..59fa585caed --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-automl-core/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlAutomlCore(Package): + """The azureml-automl-core package is a package containing functionality + used by the azureml-train-automl package.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_automl_core/azureml_automl_core-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='58ce54b01570996cda860c0c80693b8db19324785a356573f105afeaa31cae6c', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-dataprep@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-telemetry@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-core/package.py b/var/spack/repos/builtin/packages/py-azureml-core/package.py new file mode 100644 index 00000000000..3c1a74fcf98 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-core/package.py @@ -0,0 +1,52 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlCore(Package): + """The azureml-core contains functionality for creating and managing: + * Azure Machine Learning workspaces, experiments and runs; + * Machine learning compute respources; + * Models, images and web services. + """ + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_core/azureml_core-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='a0f2b0977f18fb7dcb88c314594a4a85c636a36be3d582be1cae25655fea6105', expand=False) + + extends('python') + depends_on('python@3.5:3.8', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-pytz', type=('build', 'run')) + depends_on('py-backports-tempfile', type=('build', 'run')) + depends_on('py-pathspec', type=('build', 'run')) + depends_on('py-requests@2.19.1:', type=('build', 'run')) + depends_on('py-azure-mgmt-resource@1.2.1:', type=('build', 'run')) + depends_on('py-azure-mgmt-containerregistry@2.0.0:', type=('build', 'run')) + depends_on('py-azure-mgmt-storage@1.5.0:', type=('build', 'run')) + depends_on('py-azure-mgmt-keyvault@0.40.0:', type=('build', 'run')) + depends_on('py-azure-mgmt-authorization@0.40.0:', type=('build', 'run')) + depends_on('py-azure-mgmt-network@10.0:10.999', type=('build', 'run')) + depends_on('py-azure-graphrbac@0.40.0:', type=('build', 'run')) + depends_on('py-azure-common@1.1.12:', type=('build', 'run')) + depends_on('py-msrest@0.5.1:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.33:', type=('build', 'run')) + depends_on('py-urllib3@1.23:', type=('build', 'run')) + depends_on('py-cryptography@:1.8,2.3:', type=('build', 'run')) + depends_on('py-python-dateutil@2.7.3:', type=('build', 'run')) + depends_on('py-ndg-httpsclient', type=('build', 'run')) + depends_on('py-secretstorage', type=('build', 'run')) + depends_on('py-ruamel-yaml@0.16.8:', type=('build', 'run')) + depends_on('py-jsonpickle', type=('build', 'run')) + depends_on('py-contextlib2', type=('build', 'run')) + depends_on('py-docker', type=('build', 'run')) + depends_on('py-pyjwt', type=('build', 'run')) + depends_on('py-adal@1.2.0:', type=('build', 'run')) + depends_on('py-pyopenssl', type=('build', 'run')) + depends_on('py-jmespath', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-dataprep-native/package.py b/var/spack/repos/builtin/packages/py-azureml-dataprep-native/package.py new file mode 100644 index 00000000000..afd1a432727 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-dataprep-native/package.py @@ -0,0 +1,27 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys + + +class PyAzuremlDataprepNative(Package): + """Python Package for AzureML DataPrep specific native extensions.""" + + homepage = "http://aka.ms/data-prep-sdk" + + if sys.platform == 'darwin': + version('14.2.1', sha256='0711ea6465a555d4ed052b7ecf3ed580d711ca7499a12be4c9736d5555ab2786', expand=False, + url='https://pypi.io/packages/cp37/a/azureml_dataprep_native/azureml_dataprep_native-14.2.1-cp37-cp37m-macosx_10_9_x86_64.whl') + elif sys.platform.startswith('linux'): + version('14.2.1', sha256='0817ec5c378a9bcd1af8edda511ca9d02bdc7087e6f8802c459c9b8f3fde4ade', expand=False, + url='https://pypi.io/packages/cp37/a/azureml_dataprep_native/azureml_dataprep_native-14.2.1-cp37-cp37m-manylinux1_x86_64.whl') + + extends('python') + depends_on('python@3.7.0:3.7.999', type=('build', 'run')) + depends_on('py-pip', type='build') + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-dataprep/package.py b/var/spack/repos/builtin/packages/py-azureml-dataprep/package.py new file mode 100644 index 00000000000..064ce4e0e6d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-dataprep/package.py @@ -0,0 +1,28 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlDataprep(Package): + """Azure ML Data Preparation SDK.""" + + homepage = "http://aka.ms/data-prep-sdk" + url = "https://pypi.io/packages/py3/a/azureml_dataprep/azureml_dataprep-1.8.2-py3-none-any.whl" + + version('1.8.2', sha256='e53f3206f0bd4af8d5e7de3a94c2c6e662902b86e94a7b9d930e36329fe5820f', expand=False) + + variant('fuse', default=False, description='Build with FUSE support') + + extends('python') + depends_on('python@3:', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-dotnetcore2@2.1.14:', type=('build', 'run')) + depends_on('py-azureml-dataprep-native@14.2.1:14.999', type=('build', 'run')) + depends_on('py-cloudpickle@1.1.0:', type=('build', 'run')) + depends_on('py-azure-identity@1.2.0:1.2.999', type=('build', 'run')) + depends_on('py-fusepy@3.0.1:', when='+fuse', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-pipeline-core/package.py b/var/spack/repos/builtin/packages/py-azureml-pipeline-core/package.py new file mode 100644 index 00000000000..ba7128b03e6 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-pipeline-core/package.py @@ -0,0 +1,22 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlPipelineCore(Package): + """Core functionality to enable azureml-pipeline feature.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_pipeline_core/azureml_pipeline_core-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='24e1c57a57e75f9d74ea6f45fa4e93c1ee3114c8ed9029d538f9cc8e4f8945b2', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-core@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-pipeline-steps/package.py b/var/spack/repos/builtin/packages/py-azureml-pipeline-steps/package.py new file mode 100644 index 00000000000..c461e8c8e6b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-pipeline-steps/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlPipelineSteps(Package): + """Represents a unit of computation in azureml-pipeline.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_pipeline_steps/azureml_pipeline_steps-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='3310674207ed457a26fb978e7168e400306c695f7f854f354dee9d5c7c81304c', expand=False) + + extends('python') + depends_on('python@3:', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-train-core@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-train-automl-client@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-pipeline-core@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-pipeline/package.py b/var/spack/repos/builtin/packages/py-azureml-pipeline/package.py new file mode 100644 index 00000000000..8d359f92aa6 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-pipeline/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlPipeline(Package): + """The Azure Machine Learning SDK for Python can be used to create ML + pipelines as well as to submit and track individual pipeline runs.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_pipeline/azureml_pipeline-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='43ce39789d9a255f147311e40274b5f2571c7ef3b52e218f248724ccb377a02c', expand=False) + + extends('python') + depends_on('python@3:', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-pipeline-core@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-pipeline-steps@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-sdk/package.py b/var/spack/repos/builtin/packages/py-azureml-sdk/package.py new file mode 100644 index 00000000000..fd14b0ed7b8 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-sdk/package.py @@ -0,0 +1,34 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlSdk(Package): + """Microsoft Azure Machine Learning Python SDK.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_sdk/azureml_sdk-1.8.0-py3-none-any.whl" + + maintainers = ['adamjstewart'] + + version('1.8.0', sha256='61107db1403ce2c1a12064eb0fa31a1d075debbf32dd17cb93b7639b615b7839', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + + # For version 1.8.0, dependencies are locked to ~= 1.8.0 + for ver in ['1.8.0']: + ver = Version(ver) + for dep in [ + 'core', 'train', 'train-automl-client', 'pipeline', + 'dataprep+fuse' + ]: + depends_on('py-azureml-{0}@{1}:{2}.999'.format( + dep, ver, ver.up_to(2) + ), when='@{0}'.format(ver), type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-telemetry/package.py b/var/spack/repos/builtin/packages/py-azureml-telemetry/package.py new file mode 100644 index 00000000000..4ceaf333999 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-telemetry/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlTelemetry(Package): + """Machine learning (ML) telemetry package is used to collect telemetry + data.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_telemetry/azureml_telemetry-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='de657efe9773bea0de76c432cbab34501ac28606fe1b380d6883562ebda3d804', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-applicationinsights', type=('build', 'run')) + depends_on('py-azureml-core@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-train-automl-client/package.py b/var/spack/repos/builtin/packages/py-azureml-train-automl-client/package.py new file mode 100644 index 00000000000..739af7a07f3 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-train-automl-client/package.py @@ -0,0 +1,27 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlTrainAutomlClient(Package): + """The azureml-train-automl-client package contains functionality for + automatically finding the best machine learning model and its parameters, + given training and test data.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_train_automl_client/azureml_train_automl_client-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='562300095db6c4dea7b052e255c53dd95c4c3d0589a828b545497fe1ca7e9677', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-dataprep@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-automl-core@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-core@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-telemetry@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-train-core/package.py b/var/spack/repos/builtin/packages/py-azureml-train-core/package.py new file mode 100644 index 00000000000..55e706762a9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-train-core/package.py @@ -0,0 +1,26 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlTrainCore(Package): + """The azureml-train-core contains functionality used by azureml-train + metapackage.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_train_core/azureml_train_core-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='5a8d90a08d4477527049d793feb40d07dc32fafc0e4e57b4f0729d3c50b408a2', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-train-restclients-hyperdrive@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-core@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-azureml-telemetry@1.8.0:1.8.999', type=('build', 'run')) + depends_on('py-flake8@3.1.0:3.7.9', when='^python@3.6:', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-train-restclients-hyperdrive/package.py b/var/spack/repos/builtin/packages/py-azureml-train-restclients-hyperdrive/package.py new file mode 100644 index 00000000000..4bcf121cc85 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-train-restclients-hyperdrive/package.py @@ -0,0 +1,25 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlTrainRestclientsHyperdrive(Package): + """The azureml-train-restclients-hyperdrive contains functionality for + azureml-train metapackage.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_train_restclients_hyperdrive/azureml_train_restclients_hyperdrive-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='1633c7eb0fd96714f54f72072ccf1c5ee1ef0a8ba52680793f20d27e0fd43c87', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-requests@2.19.1:', type=('build', 'run')) + depends_on('py-msrest@0.5.1:', type=('build', 'run')) + depends_on('py-msrestazure@0.4.33:', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-azureml-train/package.py b/var/spack/repos/builtin/packages/py-azureml-train/package.py new file mode 100644 index 00000000000..c285968349d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-azureml-train/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyAzuremlTrain(Package): + """The azureml-train package provides estimators for training models using + different deep learning frameworks and functionality for hyperparameter + tuning using Azure cloud.""" + + homepage = "https://docs.microsoft.com/en-us/azure/machine-learning/service/" + url = "https://pypi.io/packages/py3/a/azureml_train/azureml_train-1.8.0-py3-none-any.whl" + + version('1.8.0', sha256='124e5b7d8d64bac61db022f305bd31c25e57fdcb4be93eefd4244a04a13deab3', expand=False) + + extends('python') + depends_on('python@3.5:3.999', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-azureml-train-core@1.8.0:1.8.999', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-distro/package.py b/var/spack/repos/builtin/packages/py-distro/package.py new file mode 100644 index 00000000000..4bda8e40859 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-distro/package.py @@ -0,0 +1,15 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyDistro(PythonPackage): + """Distro - an OS platform information API.""" + + homepage = "https://github.com/nir0s/distro" + url = "https://pypi.io/packages/source/d/distro/distro-1.5.0.tar.gz" + + version('1.5.0', sha256='0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92') + + depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-docker/package.py b/var/spack/repos/builtin/packages/py-docker/package.py new file mode 100644 index 00000000000..148da74a954 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-docker/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyDocker(PythonPackage): + """A Python library for the Docker Engine API.""" + + homepage = "https://github.com/docker/docker-py" + url = "https://pypi.io/packages/source/d/docker/docker-4.2.1.tar.gz" + + version('4.2.1', sha256='380a20d38fbfaa872e96ee4d0d23ad9beb0f9ed57ff1c30653cbeb0c9c0964f2') + + depends_on('python@2.7:2.8,3.5:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-six@1.4.0:', type=('build', 'run')) + depends_on('py-websocket-client@0.32.0:', type=('build', 'run')) + depends_on('py-requests@2.14.2:2.17.999,2.18.1:', type=('build', 'run')) + depends_on('py-backports-ssl-match-hostname@3.5:', when='^python@:3.4', type=('build', 'run')) + depends_on('py-ipaddress@1.0.16:', when='^python@:3.2', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-dotnetcore2/package.py b/var/spack/repos/builtin/packages/py-dotnetcore2/package.py new file mode 100644 index 00000000000..2a33149cccf --- /dev/null +++ b/var/spack/repos/builtin/packages/py-dotnetcore2/package.py @@ -0,0 +1,28 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys + + +class PyDotnetcore2(Package): + """.Net Core 2.1 runtime.""" + + homepage = "https://github.com/dotnet/core" + + if sys.platform == 'darwin': + version('2.1.14', sha256='68182f4b704db401b2012c10ed8a19561f8d487063632f8731c2e58960ca9242', expand=False, + url='https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-macosx_10_9_x86_64.whl') + elif sys.platform.startswith('linux'): + version('2.1.14', sha256='d8d83ac30c22a0e48a9a881e117d98da17f95c4098cb9500a35e323b8e4ab737', expand=False, + url='https://pypi.io/packages/py3/d/dotnetcore2/dotnetcore2-2.1.14-py3-none-manylinux1_x86_64.whl') + + extends('python') + depends_on('python@3:', type=('build', 'run')) + depends_on('py-pip', type='build') + depends_on('py-distro@1.2.0:', type=('build', 'run')) + + def install(self, spec, prefix): + pip = which('pip') + pip('install', self.stage.archive_file, '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-fusepy/package.py b/var/spack/repos/builtin/packages/py-fusepy/package.py index 7ca452b5380..c9df4ba5ba0 100644 --- a/var/spack/repos/builtin/packages/py-fusepy/package.py +++ b/var/spack/repos/builtin/packages/py-fusepy/package.py @@ -11,8 +11,9 @@ class PyFusepy(PythonPackage): MacFUSE. It's just one file and is implemented using ctypes.""" homepage = "https://github.com/fusepy/fusepy" - url = "https://github.com/fusepy/fusepy/archive/v2.0.4.tar.gz" + url = "https://pypi.io/packages/source/f/fusepy/fusepy-3.0.1.tar.gz" - version('2.0.4', sha256='802610ab25ad04fc9ef34d024a0abe41cdcaff6a2cb8b2fb92cdda0057c09d1f') + version('3.0.1', sha256='72ff783ec2f43de3ab394e3f7457605bf04c8cf288a2f4068b4cde141d4ee6bd') + version('2.0.4', sha256='10f5c7f5414241bffecdc333c4d3a725f1d6605cae6b4eaf86a838ff49cdaf6c') depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-jeepney/package.py b/var/spack/repos/builtin/packages/py-jeepney/package.py new file mode 100644 index 00000000000..c44cc4c3d5d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-jeepney/package.py @@ -0,0 +1,15 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyJeepney(PythonPackage): + """Low-level, pure Python DBus protocol wrapper.""" + + homepage = "https://gitlab.com/takluyver/jeepney" + url = "https://pypi.io/packages/source/j/jeepney/jeepney-0.4.3.tar.gz" + + version('0.4.3', sha256='3479b861cc2b6407de5188695fa1a8d57e5072d7059322469b62628869b8e36e') + + depends_on('python@3.5:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-jsonpickle/package.py b/var/spack/repos/builtin/packages/py-jsonpickle/package.py new file mode 100644 index 00000000000..7bc9aeda58e --- /dev/null +++ b/var/spack/repos/builtin/packages/py-jsonpickle/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyJsonpickle(PythonPackage): + """Python library for serializing any arbitrary object graph into JSON.""" + + homepage = "https://github.com/jsonpickle/jsonpickle" + url = "https://pypi.io/packages/source/j/jsonpickle/jsonpickle-1.4.1.tar.gz" + + version('1.4.1', sha256='e8d4b7cd0bd6826001a74377df1079a76ad8bae0f909282de2554164c837c8ba') + + depends_on('python@2.7:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-setuptools-scm@3.4.1:+toml', type='build') + depends_on('py-importlib-metadata', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-msal-extensions/package.py b/var/spack/repos/builtin/packages/py-msal-extensions/package.py new file mode 100644 index 00000000000..75e6740f359 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-msal-extensions/package.py @@ -0,0 +1,23 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyMsalExtensions(PythonPackage): + """The Microsoft Authentication Extensions for Python offers secure + mechanisms for client applications to perform cross-platform token cache + serialization and persistence. It gives additional support to the + Microsoft Authentication Library for Python (MSAL).""" + + homepage = "https://github.com/AzureAD/microsoft-authentication-library-for-python" + url = "https://pypi.io/packages/source/m/msal-extensions/msal-extensions-0.2.2.tar.gz" + + version('0.2.2', sha256='31414753c484679bb3b6c6401623eb4c3ccab630af215f2f78c1d5c4f8e1d1a9') + version('0.1.3', sha256='59e171a9a4baacdbf001c66915efeaef372fb424421f1a4397115a3ddd6205dc') + + depends_on('py-setuptools', type='build') + depends_on('py-msal@0.4.1:1.999', type=('build', 'run')) + depends_on('py-portalocker@1.0:1.999', type=('build', 'run')) + depends_on('py-pathlib2', when='@0.2:^python@:2', type=('build', 'run')) + depends_on('py-pytest', type='test') diff --git a/var/spack/repos/builtin/packages/py-msal/package.py b/var/spack/repos/builtin/packages/py-msal/package.py new file mode 100644 index 00000000000..ade131afc8d --- /dev/null +++ b/var/spack/repos/builtin/packages/py-msal/package.py @@ -0,0 +1,20 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyMsal(PythonPackage): + """The Microsoft Authentication Library (MSAL) for Python library enables + your app to access the Microsoft Cloud by supporting authentication of + users with Microsoft Azure Active Directory accounts (AAD) and Microsoft + Accounts (MSA) using industry standard OAuth2 and OpenID Connect.""" + + homepage = "https://github.com/AzureAD/microsoft-authentication-library-for-python" + url = "https://pypi.io/packages/source/m/msal/msal-1.3.0.tar.gz" + + version('1.3.0', sha256='5442a3a9d006506e653d3c4daff40538bdf067bf07b6b73b32d1b231d5e77a92') + + depends_on('py-setuptools', type='build') + depends_on('py-requests@2.0.0:2.999', type=('build', 'run')) + depends_on('py-pyjwt@1.0.0:1.999+crypto', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-msrest/package.py b/var/spack/repos/builtin/packages/py-msrest/package.py new file mode 100644 index 00000000000..f589d152e79 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-msrest/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyMsrest(PythonPackage): + """AutoRest swagger generator Python client runtime.""" + + homepage = "https://github.com/Azure/msrest-for-python" + url = "https://pypi.io/packages/source/m/msrest/msrest-0.6.16.tar.gz" + + version('0.6.16', sha256='214c5be98954cb45feb6a6a858a7ae6d41a664e80294b65db225bbaa33d9ca3c') + + depends_on('py-setuptools', type='build') + depends_on('py-requests@2.16:2.999', type=('build', 'run')) + depends_on('py-requests-oauthlib@0.5.0:', type=('build', 'run')) + depends_on('py-isodate@0.6.0:', type=('build', 'run')) + depends_on('py-certifi@2017.4.17:', type=('build', 'run')) + depends_on('py-enum34@1.0.4:', when='^python@:3.3', type=('build', 'run')) + depends_on('py-typing', when='^python@:3.4', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-msrestazure/package.py b/var/spack/repos/builtin/packages/py-msrestazure/package.py new file mode 100644 index 00000000000..40208cfa670 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-msrestazure/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyMsrestazure(PythonPackage): + """AutoRest swagger generator Python client runtime. + Azure-specific module.""" + + homepage = "https://github.com/Azure/msrestazure-for-python" + url = "https://pypi.io/packages/source/m/msrestazure/msrestazure-0.6.3.tar.gz" + + version('0.6.3', sha256='0ec9db93eeea6a6cf1240624a04f49cd8bbb26b98d84a63a8220cfda858c2a96') + + depends_on('py-setuptools', type='build') + depends_on('py-msrest@0.6.0:1.999', type=('build', 'run')) + depends_on('py-adal@0.6.0:1.999', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-ndg-httpsclient/package.py b/var/spack/repos/builtin/packages/py-ndg-httpsclient/package.py new file mode 100644 index 00000000000..91a815ce99b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ndg-httpsclient/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyNdgHttpsclient(PythonPackage): + """Provides enhanced HTTPS support for httplib and urllib2 using + PyOpenSSL.""" + + homepage = "https://github.com/cedadev/ndg_httpsclient/" + url = "https://pypi.io/packages/source/n/ndg_httpsclient/ndg_httpsclient-0.5.1.tar.gz" + + version('0.5.1', sha256='d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210') + + depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-pyopenssl', type=('build', 'run')) + depends_on('py-pyasn1@0.1.1:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-pyjwt/package.py b/var/spack/repos/builtin/packages/py-pyjwt/package.py index 70ee8da7378..a8006ef623a 100644 --- a/var/spack/repos/builtin/packages/py-pyjwt/package.py +++ b/var/spack/repos/builtin/packages/py-pyjwt/package.py @@ -14,8 +14,11 @@ class PyPyjwt(PythonPackage): version('1.7.1', sha256='8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96') + variant('crypto', default=False, description='Build with cryptography support') + + depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) depends_on('py-setuptools', type='build') + depends_on('py-cryptography@1.4:', when='+crypto', type=('build', 'run')) depends_on('py-pytest@4.0.1:4.99.99', type='test') depends_on('py-pytest-cov@2.6.0:2.99.99', type='test') depends_on('py-pytest-runner@4.2:4.99', type='test') - depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-ruamel-ordereddict/package.py b/var/spack/repos/builtin/packages/py-ruamel-ordereddict/package.py new file mode 100644 index 00000000000..88cdd59914c --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ruamel-ordereddict/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyRuamelOrdereddict(PythonPackage): + """This is an implementation of an ordered dictionary with Key Insertion + Order (KIO: updates of values do not affect the position of the key), Key + Value Insertion Order (KVIO, an existing key's position is removed and put + at the back). The standard library module OrderedDict, implemented later, + implements a subset of ordereddict functionality.""" + + homepage = "https://bitbucket.org/ruamel/ordereddict" + url = "https://pypi.io/packages/source/r/ruamel.ordereddict/ruamel.ordereddict-0.4.14.tar.gz" + + version('0.4.14', sha256='281051d26eb2b18ef3d920e1e260716a52bd058a6b1a2f324102fc6a15cb8d4a') + + depends_on('py-setuptools', type='build') diff --git a/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py b/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py new file mode 100644 index 00000000000..9724bff0aff --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ruamel-yaml-clib/package.py @@ -0,0 +1,17 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyRuamelYamlClib(PythonPackage): + """C version of reader, parser and emitter for ruamel.yaml derived from + libyaml.""" + + homepage = "https://bitbucket.org/ruamel/yaml.clib" + url = "https://pypi.io/packages/source/r/ruamel.yaml.clib/ruamel.yaml.clib-0.2.0.tar.gz" + + version('0.2.0', sha256='b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c') + + depends_on('python@2.7:2.8,3.5:', type=('build', 'run')) + depends_on('py-setuptools@28.7.0:', type='build') diff --git a/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py b/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py index 9c3ba760384..68e9348936b 100644 --- a/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py +++ b/var/spack/repos/builtin/packages/py-ruamel-yaml/package.py @@ -12,13 +12,17 @@ class PyRuamelYaml(PythonPackage): seq/map flow style, and map key order """ - homepage = "https://bitbucket.org/ruamel/yaml/src/default/" - url = "https://pypi.io/packages/source/r/ruamel.yaml/ruamel.yaml-0.16.5.tar.gz" + homepage = "https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree" + url = "https://pypi.io/packages/source/r/ruamel.yaml/ruamel.yaml-0.16.10.tar.gz" - version('0.16.5', sha256='412a6f5cfdc0525dee6a27c08f5415c7fd832a7afcb7a0ed7319628aed23d408') - version('0.11.7', sha256='c89363e16c9eafb9354e55d757723efeff8682d05e56b0881450002ffb00a344') + version('0.16.10', sha256='099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954') + version('0.16.5', sha256='412a6f5cfdc0525dee6a27c08f5415c7fd832a7afcb7a0ed7319628aed23d408') + version('0.11.7', sha256='c89363e16c9eafb9354e55d757723efeff8682d05e56b0881450002ffb00a344') + depends_on('python@2.7:2.8,3.5:', type=('build', 'run')) depends_on('py-setuptools', type='build') + depends_on('py-ruamel-ordereddict', when='@0.15:^python@:2', type=('build', 'run')) + depends_on('py-ruamel-yaml-clib@0.1.2:', when='@0.16:^python@:3.8', type=('build', 'run')) @run_after('install') def fix_import_error(self): diff --git a/var/spack/repos/builtin/packages/py-secretstorage/package.py b/var/spack/repos/builtin/packages/py-secretstorage/package.py new file mode 100644 index 00000000000..36be45cd0ec --- /dev/null +++ b/var/spack/repos/builtin/packages/py-secretstorage/package.py @@ -0,0 +1,18 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PySecretstorage(PythonPackage): + """Python bindings to FreeDesktop.org Secret Service API.""" + + homepage = "https://github.com/mitya57/secretstorage" + url = "https://pypi.io/packages/source/S/SecretStorage/SecretStorage-3.1.2.tar.gz" + + version('3.1.2', sha256='15da8a989b65498e29be338b3b279965f1b8f09b9668bd8010da183024c8bff6') + + depends_on('python@3.5:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-cryptography', type=('build', 'run')) + depends_on('py-jeepney@0.4.2:', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-setuptools-scm/package.py b/var/spack/repos/builtin/packages/py-setuptools-scm/package.py index b91bc3e3248..eb157f8ef51 100644 --- a/var/spack/repos/builtin/packages/py-setuptools-scm/package.py +++ b/var/spack/repos/builtin/packages/py-setuptools-scm/package.py @@ -10,13 +10,18 @@ class PySetuptoolsScm(PythonPackage): """The blessed package to manage your versions by scm tags.""" homepage = "https://github.com/pypa/setuptools_scm" - url = "https://pypi.io/packages/source/s/setuptools_scm/setuptools_scm-3.3.3.tar.gz" + url = "https://pypi.io/packages/source/s/setuptools_scm/setuptools_scm-4.1.2.tar.gz" import_modules = ['setuptools_scm'] + version('4.1.2', sha256='a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8') version('3.3.3', sha256='bd25e1fb5e4d603dcf490f1fde40fb4c595b357795674c3e5cb7f6217ab39ea5') version('3.1.0', sha256='1191f2a136b5e86f7ca8ab00a97ef7aef997131f1f6d4971be69a1ef387d8b40') version('1.15.6', sha256='49ab4685589986a42da85706b3311a2f74f1af567d39fee6cb1e088d7a75fb5f') - depends_on('py-setuptools', type='build') + variant('toml', default=False, description='Build with TOML support') + depends_on('python@2.7:2.8,3.4:', type=('build', 'run')) + depends_on('python@2.7:2.8,3.5:', when='@4:', type=('build', 'run')) + depends_on('py-setuptools@34.4:', type=('build', 'run')) + depends_on('py-toml', when='+toml', type=('build', 'run')) diff --git a/var/spack/repos/builtin/packages/py-websocket-client/package.py b/var/spack/repos/builtin/packages/py-websocket-client/package.py new file mode 100644 index 00000000000..75ce5e69cc2 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-websocket-client/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +class PyWebsocketClient(PythonPackage): + """WebSocket client for Python. hybi13 is supported.""" + + homepage = "https://github.com/websocket-client/websocket-client.git" + url = "https://pypi.io/packages/source/w/websocket_client/websocket_client-0.57.0.tar.gz" + + version('0.57.0', sha256='d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010') + + depends_on('python@2.6:2.8,3.4:', type=('build', 'run')) + depends_on('py-setuptools', type='build') + depends_on('py-six', type=('build', 'run')) + depends_on('py-backports-ssl-match-hostname', when='^python@2.6:2.7.9', type=('build', 'run')) + depends_on('py-argparse', when='^python@:2.6', type=('build', 'run')) From c401c63156b362fda12f5bf6d8b95a99c2540ad1 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 26 Jun 2020 04:20:09 +0200 Subject: [PATCH 36/38] Allow detection for "-classic" version of CCE (#17256) * cce: detect modules based on the classic backend * cce: tweaked version checks for clang based options * Added unit test for cce flags --- lib/spack/spack/compilers/cce.py | 15 ++++++++++----- lib/spack/spack/operating_systems/cray_backend.py | 4 +++- lib/spack/spack/test/compilers/basics.py | 8 ++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py index 47c0263cfd7..9844eb299d1 100644 --- a/lib/spack/spack/compilers/cce.py +++ b/lib/spack/spack/compilers/cce.py @@ -32,9 +32,14 @@ class Cce(Compiler): 'f77': 'cce/ftn', 'fc': 'cce/ftn'} + @property + def is_clang_based(self): + version = self.version + return version >= ver('9.0') and 'classic' not in str(version) + @property def version_argument(self): - if self.version >= ver('9.0'): + if self.is_clang_based: return '--version' return '-V' @@ -50,19 +55,19 @@ def debug_flags(self): @property def openmp_flag(self): - if self.version >= ver('9.0'): + if self.is_clang_based: return '-fopenmp' return "-h omp" @property def cxx11_flag(self): - if self.version >= ver('9.0'): + if self.is_clang_based: return '-std=c++11' return "-h std=c++11" @property def c99_flag(self): - if self.version >= ver('9.0'): + if self.is_clang_based: return '-std=c99' elif self.version >= ver('8.4'): return '-h std=c99,noconform,gnu' @@ -75,7 +80,7 @@ def c99_flag(self): @property def c11_flag(self): - if self.version >= ver('9.0'): + if self.is_clang_based: return '-std=c11' elif self.version >= ver('8.5'): return '-h std=c11,noconform,gnu' diff --git a/lib/spack/spack/operating_systems/cray_backend.py b/lib/spack/spack/operating_systems/cray_backend.py index 91c0e6ae985..262df6a7bdc 100644 --- a/lib/spack/spack/operating_systems/cray_backend.py +++ b/lib/spack/spack/operating_systems/cray_backend.py @@ -142,7 +142,9 @@ def detect_version(self, detect_version_args): compiler_name = detect_version_args.id.compiler_name compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) output = modulecmd('avail', compiler_cls.PrgEnv_compiler) - version_regex = r'(%s)/([\d\.]+[\d])' % compiler_cls.PrgEnv_compiler + version_regex = r'({0})/([\d\.]+[\d]-?[\w]*)'.format( + compiler_cls.PrgEnv_compiler + ) matches = re.findall(version_regex, output) version = tuple(version for _, version in matches) compiler_id = detect_version_args.id diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 825975b21aa..9ccf44f14f7 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -358,12 +358,20 @@ def test_arm_flags(): def test_cce_flags(): + supported_flag_test("version_argument", "--version", "cce@9.0.1") + supported_flag_test("version_argument", "-V", "cce@9.0.1-classic") + supported_flag_test("openmp_flag", "-fopenmp", "cce@9.0.1") + supported_flag_test("openmp_flag", "-h omp", "cce@9.0.1-classic") supported_flag_test("openmp_flag", "-h omp", "cce@1.0") + supported_flag_test("cxx11_flag", "-std=c++11", "cce@9.0.1") + supported_flag_test("cxx11_flag", "-h std=c++11", "cce@9.0.1-classic") supported_flag_test("cxx11_flag", "-h std=c++11", "cce@1.0") unsupported_flag_test("c99_flag", "cce@8.0") + supported_flag_test("c99_flag", "-std=c99", "cce@9.0.1") supported_flag_test("c99_flag", "-h c99,noconform,gnu", "cce@8.1") supported_flag_test("c99_flag", "-h std=c99,noconform,gnu", "cce@8.4") unsupported_flag_test("c11_flag", "cce@8.4") + supported_flag_test("c11_flag", "-std=c11", "cce@9.0.1") supported_flag_test("c11_flag", "-h std=c11,noconform,gnu", "cce@8.5") supported_flag_test("cc_pic_flag", "-h PIC", "cce@1.0") supported_flag_test("cxx_pic_flag", "-h PIC", "cce@1.0") From ec108dbebdbb00ab517dd8f84dbb976d6078a93a Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Fri, 26 Jun 2020 09:57:22 -0500 Subject: [PATCH 37/38] Allow `spack remove -f` and `spack uninstall` to work on matrices (#17222) * Allow `spack remove -f` and `spack uninstall` to work on matrices Allow Environment.remove(force=True) to remove the concrete spec from the environment even when the user spec cannot be removed because it is in a matrix. --- lib/spack/spack/environment.py | 20 +++++++++++---- lib/spack/spack/test/cmd/env.py | 45 +++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 8eab70b544d..5ec683f6b2b 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -892,13 +892,23 @@ def remove(self, query_spec, list_name=user_speclist_name, force=False): old_specs = set(self.user_specs) for spec in matches: if spec in list_to_change: - list_to_change.remove(spec) - - self.update_stale_references(list_name) + try: + list_to_change.remove(spec) + self.update_stale_references(list_name) + new_specs = set(self.user_specs) + except spack.spec_list.SpecListError: + # define new specs list + new_specs = set(self.user_specs) + msg = "Spec '%s' is part of a spec matrix and " % spec + msg += "cannot be removed from list '%s'." % list_to_change + if force: + msg += " It will be removed from the concrete specs." + # Mock new specs so we can remove this spec from + # concrete spec lists + new_specs.remove(spec) + tty.warn(msg) # If force, update stale concretized specs - # Only check specs removed by this operation - new_specs = set(self.user_specs) for spec in old_specs - new_specs: if force and spec in self.concretized_user_specs: i = self.concretized_user_specs.index(spec) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 46977d8eec3..b1cc9ce8b5b 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -19,7 +19,6 @@ from spack.main import SpackCommand from spack.stage import stage_prefix -from spack.spec_list import SpecListError from spack.util.mock_package import MockPackageMultiRepo import spack.util.spack_json as sjson from spack.util.path import substitute_path_variables @@ -1234,7 +1233,7 @@ def test_stack_yaml_remove_from_list_force(tmpdir): assert Spec('callpath ^mpich') in test.user_specs -def test_stack_yaml_attempt_remove_from_matrix(tmpdir): +def test_stack_yaml_remove_from_matrix_no_effect(tmpdir): filename = str(tmpdir.join('spack.yaml')) with open(filename, 'w') as f: f.write("""\ @@ -1249,9 +1248,45 @@ def test_stack_yaml_attempt_remove_from_matrix(tmpdir): """) with tmpdir.as_cwd(): env('create', 'test', './spack.yaml') - with pytest.raises(SpecListError): - with ev.read('test'): - remove('-l', 'packages', 'mpileaks') + with ev.read('test') as e: + before = e.user_specs.specs + remove('-l', 'packages', 'mpileaks') + after = e.user_specs.specs + + assert before == after + + +def test_stack_yaml_force_remove_from_matrix(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: + - matrix: + - [mpileaks, callpath] + - [target=be] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test') as e: + concretize() + + before_user = e.user_specs.specs + before_conc = e.concretized_user_specs + + remove('-f', '-l', 'packages', 'mpileaks') + + after_user = e.user_specs.specs + after_conc = e.concretized_user_specs + + assert before_user == after_user + + mpileaks_spec = Spec('mpileaks target=be') + assert mpileaks_spec in before_conc + assert mpileaks_spec not in after_conc def test_stack_concretize_extraneous_deps(tmpdir, config, mock_packages): From 1602b7a56192c23fb7fc110a1c8a17ec0837bdd3 Mon Sep 17 00:00:00 2001 From: Jeffrey Salmond Date: Fri, 26 Jun 2020 16:01:12 +0100 Subject: [PATCH 38/38] add environment-awareness to `buildcache create` (#16580) * add buildcache create test * add functionality and test to create buildcache from environment * use env.concretized_user_specs rather than env.roots to get concretized specs, as suggested in review from becker33 --- lib/spack/spack/cmd/buildcache.py | 8 +++- lib/spack/spack/test/cmd/buildcache.py | 51 ++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index ee00f6259f3..29048d57875 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -338,10 +338,14 @@ def _createtarball(env, spec_yaml, packages, add_spec, add_deps, elif packages: packages = packages + elif env: + packages = env.concretized_user_specs + else: tty.die("build cache file creation requires at least one" + - " installed package argument or else path to a" + - " yaml file containing a spec to install") + " installed package spec, an activate environment," + + " or else a path to a yaml file containing a spec" + + " to install") pkgs = set(packages) specs = set() diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index 03d09b9771c..af02d2d0977 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -5,14 +5,19 @@ import errno import platform +import os import pytest import spack.main import spack.binary_distribution +import spack.environment as ev +from spack.spec import Spec buildcache = spack.main.SpackCommand('buildcache') install = spack.main.SpackCommand('install') +env = spack.main.SpackCommand('env') +add = spack.main.SpackCommand('add') @pytest.fixture() @@ -45,6 +50,52 @@ def test_buildcache_list_duplicates(mock_get_specs, capsys): assert output.count('mpileaks') == 3 +def tests_buildcache_create( + install_mockery, mock_fetch, monkeypatch, tmpdir): + """"Ensure that buildcache create creates output files""" + pkg = 'trivial-install-test-package' + install(pkg) + + buildcache('create', '-d', str(tmpdir), '--unsigned', pkg) + + spec = Spec(pkg).concretized() + tarball_path = spack.binary_distribution.tarball_path_name(spec, '.spack') + tarball = spack.binary_distribution.tarball_name(spec, '.spec.yaml') + assert os.path.exists( + os.path.join(str(tmpdir), 'build_cache', tarball_path)) + assert os.path.exists( + os.path.join(str(tmpdir), 'build_cache', tarball)) + + +def tests_buildcache_create_env( + install_mockery, mock_fetch, monkeypatch, + tmpdir, mutable_mock_env_path): + """"Ensure that buildcache create creates output files from env""" + pkg = 'trivial-install-test-package' + + env('create', 'test') + with ev.read('test'): + add(pkg) + install() + + buildcache('create', '-d', str(tmpdir), '--unsigned') + + spec = Spec(pkg).concretized() + tarball_path = spack.binary_distribution.tarball_path_name(spec, '.spack') + tarball = spack.binary_distribution.tarball_name(spec, '.spec.yaml') + assert os.path.exists( + os.path.join(str(tmpdir), 'build_cache', tarball_path)) + assert os.path.exists( + os.path.join(str(tmpdir), 'build_cache', tarball)) + + +def test_buildcache_create_fails_on_noargs(tmpdir): + """Ensure that buildcache create fails when given no args or + environment.""" + with pytest.raises(spack.main.SpackCommandError): + buildcache('create', '-d', str(tmpdir), '--unsigned') + + def test_buildcache_create_fail_on_perm_denied( install_mockery, mock_fetch, monkeypatch, tmpdir): """Ensure that buildcache create fails on permission denied error."""