From 44681dbca51447142e28b8bf941ba74a7a08c3eb Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Thu, 2 Jul 2020 09:45:41 +0200 Subject: [PATCH 01/10] autotools: Fix config.guess detection, take two (#17333) The previous fix from #17149 contained a thinko that produced errors for packages that overwrite configure_directory. --- lib/spack/spack/build_systems/autotools.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index 27a3ee7657a..f07a2169ac0 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -118,17 +118,15 @@ def _do_patch_config_files(self): config_file = 'config.{0}'.format(config_name) if os.path.exists(config_file): # First search the top-level source directory - my_config_files[config_name] = os.path.join( - self.configure_directory, config_file) + my_config_files[config_name] = os.path.abspath(config_file) else: # Then search in all sub directories recursively. # We would like to use AC_CONFIG_AUX_DIR, but not all packages # ship with their configure.in or configure.ac. config_path = next((os.path.join(r, f) - for r, ds, fs in os.walk( - self.configure_directory) for f in fs + for r, ds, fs in os.walk('.') for f in fs if f == config_file), None) - my_config_files[config_name] = config_path + my_config_files[config_name] = os.path.abspath(config_path) if my_config_files[config_name] is not None: try: From 7717f00dacdaa3ace12b0d4408883361327e2f75 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sun, 5 Jul 2020 11:09:24 -0500 Subject: [PATCH 02/10] Fix Intel MPI super invocation, again (#17378) --- var/spack/repos/builtin/packages/intel-mpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/intel-mpi/package.py b/var/spack/repos/builtin/packages/intel-mpi/package.py index 57d173bc950..7c87139a7bf 100644 --- a/var/spack/repos/builtin/packages/intel-mpi/package.py +++ b/var/spack/repos/builtin/packages/intel-mpi/package.py @@ -65,7 +65,7 @@ def setup_dependent_build_environment(self, *args): }) def setup_run_environment(self, env): - super(IntelMPI, self).setup_run_environment(env) + super(IntelMpi, self).setup_run_environment(env) for name, value in self.mpi_compiler.wrappers.items(): env.set(name, value) From 28549f300d9c69ee7386f7b54fc1330d0e9652bd Mon Sep 17 00:00:00 2001 From: TZ Date: Sun, 5 Jul 2020 18:10:28 +0200 Subject: [PATCH 03/10] inel-mpi: fix for wrong structure name instroduced in ea8a0be4 (#17382) it's mpi_compiler_wrappers and not mpi_compiler._wrappers fixes 2nd part of #17371 --- var/spack/repos/builtin/packages/intel-mpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/intel-mpi/package.py b/var/spack/repos/builtin/packages/intel-mpi/package.py index 7c87139a7bf..65e7d6d4ce2 100644 --- a/var/spack/repos/builtin/packages/intel-mpi/package.py +++ b/var/spack/repos/builtin/packages/intel-mpi/package.py @@ -67,5 +67,5 @@ def setup_dependent_build_environment(self, *args): def setup_run_environment(self, env): super(IntelMpi, self).setup_run_environment(env) - for name, value in self.mpi_compiler.wrappers.items(): + for name, value in self.mpi_compiler_wrappers.items(): env.set(name, value) From 4e4de51f0d4ac7b08cb30e5d7de7ec726432fbca Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 6 Jul 2020 19:53:02 +0200 Subject: [PATCH 04/10] autotools bugfix: handle missing config.guess (#17356) Spack was attempting to calculate abspath on the located config.guess path even when it was not found (None); this commit skips the abspath calculation when config.guess is not found. --- lib/spack/spack/build_systems/autotools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index f07a2169ac0..1ea238e2d16 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -123,10 +123,10 @@ def _do_patch_config_files(self): # Then search in all sub directories recursively. # We would like to use AC_CONFIG_AUX_DIR, but not all packages # ship with their configure.in or configure.ac. - config_path = next((os.path.join(r, f) + config_path = next((os.path.abspath(os.path.join(r, f)) for r, ds, fs in os.walk('.') for f in fs if f == config_file), None) - my_config_files[config_name] = os.path.abspath(config_path) + my_config_files[config_name] = config_path if my_config_files[config_name] is not None: try: From c8a83661c2c761c9de5ff30ae82aa79a635e5d64 Mon Sep 17 00:00:00 2001 From: cedricchevalier19 Date: Mon, 6 Jul 2020 20:02:35 +0200 Subject: [PATCH 05/10] Fix gcc + binutils compilation. (#9024) * fix binutils deptype for gcc binutils needs to be a run dependency of gcc * Fix gcc+binutils build on RHEL7+ static-libstdc++ is not available with system gcc. Anyway, as it is for bootstraping, we do not really care depending on a shared libstdc++. Co-authored-by: Michael Kuhn --- var/spack/repos/builtin/packages/gcc/package.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 4c822b11982..3072402dabe 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -115,7 +115,7 @@ class Gcc(AutotoolsPackage, GNUMirrorPackage): depends_on('zstd', when='@10:') depends_on('iconv', when='platform=darwin') depends_on('gnat', when='languages=ada') - depends_on('binutils~libiberty', when='+binutils') + depends_on('binutils~libiberty', when='+binutils', type=('build', 'link', 'run')) depends_on('zip', type='build', when='languages=java') depends_on('cuda', when='+nvptx') @@ -373,8 +373,6 @@ def configure_args(self): # enable appropriate bootstrapping flags stage1_ldflags = str(self.rpath_args) boot_ldflags = stage1_ldflags + ' -static-libstdc++ -static-libgcc' - if '%gcc' in spec: - stage1_ldflags = boot_ldflags options.append('--with-stage1-ldflags=' + stage1_ldflags) options.append('--with-boot-ldflags=' + boot_ldflags) From 054e0d1d1132145f7e7c00ad9408c5c9b164d4ef Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 5 Jul 2020 22:35:01 -0700 Subject: [PATCH 06/10] bugfix: no infinite recursion in setup-env.sh on Cray On Cray platforms, we rely heavily on the module system to figure out what targets, compilers, etc. are available. This unfortunately means that we shell out to the `module` command as part of platform initialization. Because we run subcommands in a shell, we can get infinite recursion if `setup-env.sh` and friends are in some init script like `.bashrc`. This fixes the infinite loop by adding guards around `setup-env.sh`, `setup-env.csh`, and `setup-env.fish`, to prevent recursive initializations of Spack. This is safe because Spack never shells out to itself, so we do not need it to be initialized in subshells. - [x] add recursion guard around `setup-env.sh` - [x] add recursion guard around `setup-env.csh` - [x] add recursion guard around `setup-env.fish` --- share/spack/setup-env.csh | 10 ++++++++++ share/spack/setup-env.fish | 9 +++++++++ share/spack/setup-env.sh | 10 ++++++++++ 3 files changed, 29 insertions(+) diff --git a/share/spack/setup-env.csh b/share/spack/setup-env.csh index edbf51e8e28..1985e023ded 100755 --- a/share/spack/setup-env.csh +++ b/share/spack/setup-env.csh @@ -12,6 +12,13 @@ # setenv SPACK_ROOT /path/to/spack # source $SPACK_ROOT/share/spack/setup-env.csh # + +# prevent infinite recursion when spack shells out (e.g., on cray for modules) +if ($?_sp_initializing) then + exit 0 +endif +setenv _sp_initializing true + if ($?SPACK_ROOT) then set _spack_source_file = $SPACK_ROOT/share/spack/setup-env.csh set _spack_share_dir = $SPACK_ROOT/share/spack @@ -38,3 +45,6 @@ else echo "ERROR: Sourcing spack setup-env.csh requires setting SPACK_ROOT to " echo " the root of your spack installation." endif + +# done: unset sentinel variable as we're no longer initializing +unsetenv _sp_initializing diff --git a/share/spack/setup-env.fish b/share/spack/setup-env.fish index a534f09ee9a..dd2a17f74f5 100755 --- a/share/spack/setup-env.fish +++ b/share/spack/setup-env.fish @@ -36,6 +36,12 @@ # to come up with a user-friendly naming scheme for spack dotfiles. ################################################################################# +# prevent infinite recursion when spack shells out (e.g., on cray for modules) +if test -n "$_sp_initializing" + exit 0 +end +set -x _sp_initializing true + # # Test for STDERR-NOCARET feature: if this is off, fish will redirect stderr to @@ -721,3 +727,6 @@ sp_multi_pathadd MODULEPATH $_sp_tcl_roots # [3]: When the test in the if statement fails, the `status` flag is set to 1. # `true` here manuallt resets the value of `status` to 0. Since `set` # passes `status` along, we thus avoid the function returning 1 by mistake. + +# done: unset sentinel variable as we're no longer initializing +set -e _sp_initializing diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 032247cd8f3..9d984ae1999 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -39,6 +39,12 @@ # spack module files. ######################################################################## +# prevent infinite recursion when spack shells out (e.g., on cray for modules) +if [ -n "${_sp_initializing:-}" ]; then + exit 0 +fi +export _sp_initializing=true + spack() { # Store LD_LIBRARY_PATH variables from spack shell function # This is necessary because MacOS System Integrity Protection clears @@ -357,3 +363,7 @@ _sp_multi_pathadd MODULEPATH "$_sp_tcl_roots" if [ "$_sp_shell" = bash ]; then source $_sp_share_dir/spack-completion.bash fi + +# done: unset sentinel variable as we're no longer initializing +unset _sp_initializing +export _sp_initializing From e2bec75057000887f5af7f66761527d86b7562f6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 8 Jul 2020 15:59:24 -0700 Subject: [PATCH 07/10] add public spack mirror (#17077) --- etc/spack/defaults/mirrors.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 etc/spack/defaults/mirrors.yaml diff --git a/etc/spack/defaults/mirrors.yaml b/etc/spack/defaults/mirrors.yaml new file mode 100644 index 00000000000..6afc897e61b --- /dev/null +++ b/etc/spack/defaults/mirrors.yaml @@ -0,0 +1,2 @@ +mirrors: + spack-public: https://spack-llnl-mirror.s3-us-west-2.amazonaws.com/ From afbb4a5cbade91d0b80e39ecbd98051bb8fd7628 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 9 Jul 2020 13:08:51 -0500 Subject: [PATCH 08/10] installation: skip repository metadata for externals (#16954) When Spack installs a package, it stores repository package.py files for it and all of its dependencies - any package with a Spack metadata directory in its installation prefix. It turns out this was too broad: this ends up including external packages installed by Spack (e.g. installed by another Spack instance). Currently Spack doesn't store the namespace properly for such packages, so even though the package file could be fetched from the external, Spack is unable to locate it. This commit avoids the issue by skipping any attempt to locate and copy from the package repository of externals, regardless of whether they have a Spack repo directory. --- lib/spack/spack/installer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index a2b0220a8ba..978296e0512 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -405,9 +405,14 @@ def dump_packages(spec, path): source = spack.store.layout.build_packages_path(node) source_repo_root = os.path.join(source, node.namespace) - # There's no provenance installed for the source package. Skip it. - # User can always get something current from the builtin repo. - if not os.path.isdir(source_repo_root): + # If there's no provenance installed for the package, skip it. + # If it's external, skip it because it either: + # 1) it wasn't built with Spack, so it has no Spack metadata + # 2) it was built by another Spack instance, and we do not + # (currently) use Spack metadata to associate repos with externals + # built by other Spack instances. + # Spack can always get something current from the builtin repo. + if node.external or not os.path.isdir(source_repo_root): continue # Create a source repo and get the pkg directory out of it. From c2393fe566ed5660fb11e297d14155916475bb84 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Fri, 10 Jul 2020 12:45:11 -0500 Subject: [PATCH 09/10] spack install: improve error message with no args (#17454) The error message was not updated when the behavior of Spack environments was changed to not automatically activate the local environment in #17258. The previous error message no longer makes sense. --- lib/spack/spack/cmd/install.py | 15 +++++++++++++-- lib/spack/spack/test/cmd/install.py | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 10eb3c327f8..39bcc96ce52 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -268,7 +268,7 @@ def install(parser, args, **kwargs): return if not args.spec and not args.specfiles: - # if there are no args but an active environment or spack.yaml file + # if there are no args but an active environment # then install the packages from it. env = ev.get_env(args, 'install') if env: @@ -289,7 +289,18 @@ def install(parser, args, **kwargs): env.regenerate_views() return else: - tty.die("install requires a package argument or a spack.yaml file") + msg = "install requires a package argument or active environment" + if 'spack.yaml' in os.listdir(os.getcwd()): + # There's a spack.yaml file in the working dir, the user may + # have intended to use that + msg += "\n\n" + msg += "Did you mean to install using the `spack.yaml`" + msg += " in this directory? Try: \n" + msg += " spack env activate .\n" + msg += " spack install\n" + msg += " OR\n" + msg += " spack --env . install" + tty.die(msg) if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index e4df22a6a56..9ffd166e377 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -746,3 +746,27 @@ def test_compiler_bootstrap_already_installed( # Test succeeds if it does not raise an error install('gcc@2.0') install('a%gcc@2.0') + + +def test_install_fails_no_args(tmpdir): + # ensure no spack.yaml in directory + with tmpdir.as_cwd(): + output = install(fail_on_error=False) + + # check we got the short version of the error message with no spack.yaml + assert 'requires a package argument or active environment' in output + assert 'spack env activate .' not in output + assert 'using the `spack.yaml` in this directory' not in output + + +def test_install_fails_no_args_suggests_env_activation(tmpdir): + # ensure spack.yaml in directory + tmpdir.ensure('spack.yaml') + + with tmpdir.as_cwd(): + output = install(fail_on_error=False) + + # check we got the long version of the error message with spack.yaml + assert 'requires a package argument or active environment' in output + assert 'spack env activate .' in output + assert 'using the `spack.yaml` in this directory' in output From 1741279f16ffaae130142d08244c695c98e843a4 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Thu, 9 Jul 2020 16:42:08 -0700 Subject: [PATCH 10/10] Bump version to 0.15.1; update CHANGELOG and version references --- CHANGELOG.md | 16 ++++++++++++++++ lib/spack/spack/__init__.py | 2 +- lib/spack/spack/container/images.json | 12 ++++++++---- lib/spack/spack/schema/container.py | 2 +- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4050f3f9a3..b7e4cc0ad34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# v0.15.1 (2020-07-10) + +This minor release includes several important fixes: + +* Fix shell support on Cray (#17386) +* Fix use of externals installed with other Spack instances (#16954) +* Fix gcc+binutils build (#9024) +* Fixes for usage of intel-mpi (#17378 and #17382) +* Fixes to Autotools config.guess detection (#17333 and #17356) +* Update `spack install` message to prompt user when an environment is not + explicitly activated (#17454) + +This release also adds a mirror for all sources that are +fetched in Spack (#17077). It is expected to be useful when the +official website for a Spack package is unavailable. + # v0.15.0 (2020-06-28) `v0.15.0` is a major feature release. diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 9bcf7c10bbf..ceb1e256b1a 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -5,7 +5,7 @@ #: major, minor, patch version for Spack, in a tuple -spack_version_info = (0, 15, 0) +spack_version_info = (0, 15, 1) #: String containing Spack version joined with .'s spack_version = '.'.join(str(v) for v in spack_version_info) diff --git a/lib/spack/spack/container/images.json b/lib/spack/spack/container/images.json index 84ae5543393..e4761d4b40c 100644 --- a/lib/spack/spack/container/images.json +++ b/lib/spack/spack/container/images.json @@ -12,7 +12,8 @@ "0.14.1": "0.14.1", "0.14.2": "0.14.2", "0.15": "0.15", - "0.15.0": "0.15.0" + "0.15.0": "0.15.0", + "0.15.1": "0.15.1" } }, "ubuntu:16.04": { @@ -28,7 +29,8 @@ "0.14.1": "0.14.1", "0.14.2": "0.14.2", "0.15": "0.15", - "0.15.0": "0.15.0" + "0.15.0": "0.15.0", + "0.15.1": "0.15.1" } }, "centos:7": { @@ -44,7 +46,8 @@ "0.14.1": "0.14.1", "0.14.2": "0.14.2", "0.15": "0.15", - "0.15.0": "0.15.0" + "0.15.0": "0.15.0", + "0.15.1": "0.15.1" } }, "centos:6": { @@ -60,7 +63,8 @@ "0.14.1": "0.14.1", "0.14.2": "0.14.2", "0.15": "0.15", - "0.15.0": "0.15.0" + "0.15.0": "0.15.0", + "0.15.1": "0.15.1" } } } diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index d204014dc63..36cb74a8751 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -32,7 +32,7 @@ 'enum': [ 'develop', '0.14', '0.14.0', '0.14.1', '0.14.2', - '0.15', '0.15.0', + '0.15', '0.15.0', '0.15.1', ] } },