From 8821715377147ae017d428d8c1216576de0b79e1 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Wed, 19 Aug 2015 14:18:02 -0700 Subject: [PATCH 001/339] add version-based url map for cmake, provide md5 for cmake 3.3.1 --- var/spack/packages/cmake/package.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index 9efa370c8bb..ad3b69f76f7 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -23,21 +23,30 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * +import llnl.util.tty as tty class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of tools designed to build, test and package software.""" homepage = 'https://www.cmake.org' - version('2.8.10.2', '097278785da7182ec0aea8769d06860c', - url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') - - version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', - url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') - + version('2.8.10.2', '097278785da7182ec0aea8769d06860c') + version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f') + version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') + # version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') # version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') + def url_for_version(self, version): + """Handle CMake's version-based custom URLs.""" + parts = [str(p) for p in Version(version)] + if len(parts) < 3: + tty.error("Version '%s'does not match CMake's version naming scheme (z.y.x)." % version) + version_short = ".".join(parts[:2]) + version_full = ".".join(parts) + return "http://www.cmake.org/files/v%s/cmake-%s.tar.gz" % (version_short,version_full) + + def install(self, spec, prefix): configure('--prefix=' + prefix, '--parallel=' + str(make_jobs)) From 59f89dd3bee0ee2b668554c537d1d18404108ec4 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 11:04:57 -0700 Subject: [PATCH 002/339] Allow long names in format string variables --- lib/spack/spack/spec.py | 71 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb844234..3b5d16c7a75 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1503,14 +1503,28 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): in the format string. The format strings you can provide are:: $_ Package name - $@ Version - $% Compiler - $%@ Compiler & compiler version - $+ Options - $= Architecture - $# 7-char prefix of DAG hash + $@ Version with '@' prefix + $% Compiler with '%' prefix + $%@ Compiler with '%' prefix & compiler version with '@' prefix + $+ Options + $= Architecture with '=' prefix + $# 7-char prefix of DAG hash with '-' prefix $$ $ + You can also use full-string versions, which leave off the prefixes: + + ${PACKAGE} Package name + ${VERSION} Version + ${COMPILER} Full compiler string + ${COMPILERNAME} Compiler name + ${COMPILERVER} Compiler version + ${OPTIONS} Options + ${ARCHITECTURE} Architecture + ${SHA1} Dependencies 8-char sha1 prefix + + ${SPACK_ROOT} The spack root directory + ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt + Optionally you can provide a width, e.g. $20_ for a 20-wide name. Like printf, you can provide '-' for left justification, e.g. $-20_ for a left-justified name. @@ -1526,7 +1540,8 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): color = kwargs.get('color', False) length = len(format_string) out = StringIO() - escape = compiler = False + named = escape = compiler = False + named_str = fmt = '' def write(s, c): if color: @@ -1566,9 +1581,12 @@ def write(s, c): elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': - if fmt != '': + if fmt != '%s': raise ValueError("Can't use format width with $$.") out.write('$') + elif c == '{': + named = True + named_str = '' escape = False elif compiler: @@ -1582,6 +1600,43 @@ def write(s, c): out.write(c) compiler = False + elif named: + if not c == '}': + if i == length - 1: + raise ValueError("Error: unterminated ${ in format: '%s'" + % format_string) + named_str += c + continue; + if named_str == 'PACKAGE': + write(fmt % self.name, '@') + if named_str == 'VERSION': + if self.versions and self.versions != _any_version: + write(fmt % str(self.versions), '@') + elif named_str == 'COMPILER': + if self.compiler: + write(fmt % self.compiler, '%') + elif named_str == 'COMPILERNAME': + if self.compiler: + write(fmt % self.compiler.name, '%') + elif named_str == 'COMPILERVER': + if self.compiler: + write(fmt % self.compiler.versions, '%') + elif named_str == 'OPTIONS': + if self.variants: + write(fmt % str(self.variants), '+') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % str(self.architecture), '=') + elif named_str == 'SHA1': + if self.dependencies: + out.write(fmt % str(self.dep_hash(8))) + elif named_str == 'SPACK_ROOT': + out.write(fmt % spack.prefix) + elif named_str == 'SPACK_INSTALL': + out.write(fmt % spack.install_path) + + named = False + elif c == '$': escape = True if i == length - 1: From b5c597b31864826050162b358998e240761c5d7e Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 12:19:57 -0700 Subject: [PATCH 003/339] Allow specs to be sorted based on preferred packages, versions, compilers, variants and dependencies. --- lib/spack/spack/__init__.py | 8 ++ lib/spack/spack/config.py | 12 +- lib/spack/spack/preferred_packages.py | 172 ++++++++++++++++++++++++++ lib/spack/spack/spec.py | 34 +++++ 4 files changed, 222 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/preferred_packages.py diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e00..bd8478fb98e 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -69,6 +69,14 @@ from spack.directory_layout import YamlDirectoryLayout install_layout = YamlDirectoryLayout(install_path) +# +# This controls how packages are sorted when trying to choose +# the most preferred package. More preferred packages are sorted +# first. +# +from spack.preferred_packages import PreferredPackages +pkgsort = PreferredPackages() + # # This controls how things are concretized in spack. # Replace it with a subclass if you want different diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c2..dbe225960a8 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -89,8 +89,8 @@ import os import exceptions import sys +import copy -from external.ordereddict import OrderedDict from llnl.util.lang import memoized import spack.error @@ -114,8 +114,7 @@ def __init__(self, n, f, m): _ConfigCategory('compilers', 'compilers.yaml', True) _ConfigCategory('mirrors', 'mirrors.yaml', True) -_ConfigCategory('view', 'views.yaml', True) -_ConfigCategory('order', 'orders.yaml', True) +_ConfigCategory('preferred', 'preferred.yaml', True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), @@ -156,7 +155,7 @@ def _merge_dicts(d1, d2): """Recursively merges two configuration trees, with entries in d2 taking precedence over d1""" if not d1: - return d2.copy() + return copy.copy(d2) if not d2: return d1 @@ -230,6 +229,11 @@ def get_mirror_config(): return get_config('mirrors') +def get_preferred_config(): + """Get the preferred configuration from config files""" + return get_config('preferred') + + def get_config_scope_dirname(scope): """For a scope return the config directory""" global config_scopes diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py new file mode 100644 index 00000000000..248508fe809 --- /dev/null +++ b/lib/spack/spack/preferred_packages.py @@ -0,0 +1,172 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack +from spack.version import * + +class PreferredPackages(object): + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent + + def __init__(self): + self.preferred = spack.config.get_preferred_config() + self._spec_for_pkgname_cache = {} + + #Given a package name, sort component (e.g, version, compiler, ...), and + # a second_key (used by providers), return the list + def _order_for_package(self, pkgname, component, second_key): + pkglist = [pkgname] + pkglist.append('all') + for pkg in pkglist: + if not pkg in self.preferred: + continue + orders = self.preferred[pkg] + if not type(orders) is dict: + continue + if not component in orders: + continue + order = orders[component] + if type(order) is dict: + if not second_key in order: + continue; + order = order[second_key] + if not type(order) is str: + tty.die('Expected version list in preferred config, but got %s' % str(order)) + order_list = order.split(',') + return [s.strip() for s in order_list] + return [] + + + # A generic sorting function. Given a package name and sort + # component, return less-than-0, 0, or greater-than-0 if + # a is respectively less-than, equal to, or greater than b. + def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + orderlist = self._order_for_package(pkgname, component, second_key) + a_in_list = str(a) in orderlist + b_in_list = str(b) in orderlist + if a_in_list and not b_in_list: + return -1 + elif b_in_list and not a_in_list: + return 1 + + cmp_a = None + cmp_b = None + reverse = None + if not a_in_list and not b_in_list: + cmp_a = a + cmp_b = b + reverse = -1 if reverse_natural_compare else 1 + else: + cmp_a = orderlist.index(str(a)) + cmp_b = orderlist.index(str(b)) + reverse = 1 + + if cmp_a < cmp_b: + return -1 * reverse + elif cmp_a > cmp_b: + return 1 * reverse + else: + return 0 + + + # A sorting function for specs. Similar to component_compare, but + # a and b are considered to match entries in the sorting list if they + # satisfy the list component. + def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + specs = self._spec_for_pkgname(pkgname, component, second_key) + a_index = None + b_index = None + reverse = -1 if reverse_natural_compare else 1 + for i, cspec in enumerate(specs): + if a_index == None and cspec.satisfies(a): + a_index = i + if b_index: + break + if b_index == None and cspec.satisfies(b): + b_index = i + if a_index: + break + + if a_index != None and b_index == None: return -1 + elif a_index == None and b_index != None: return 1 + elif a_index != None and b_index == a_index: return -1 * cmp(a, b) + elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) + elif a < b: return 1 * reverse + elif b < a: return -1 * reverse + else: return 0 + + + # Given a sort order specified by the pkgname/component/second_key, return + # a list of CompilerSpecs, VersionLists, or Specs for that sorting list. + def _spec_for_pkgname(self, pkgname, component, second_key): + key = (pkgname, component, second_key) + if not key in self._spec_for_pkgname_cache: + pkglist = self._order_for_package(pkgname, component, second_key) + if not pkglist: + if component in self._default_order: + pkglist = self._default_order[component] + if component == 'compiler': + self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist] + elif component == 'version': + self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist] + else: + self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] + return self._spec_for_pkgname_cache[key] + + + def provider_compare(self, pkgname, provider_str, a, b): + """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or + greater-than b. A and b are possible implementations of provider_str. + One provider is less-than another if it is preferred over the other. + For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if + mvapich should be preferred over openmpi for scorep.""" + return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) + + + def version_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if version a of pkgname is + respecively less-than, equal-to, or greater-than version b of pkgname. + One version is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'version', a, b, False, None) + + + def variant_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if variant a of pkgname is + respecively less-than, equal-to, or greater-than variant b of pkgname. + One variant is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'variant', a, b, False, None) + + + def architecture_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is + respecively less-than, equal-to, or greater-than architecture b of pkgname. + One architecture is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'architecture', a, b, False, None) + + + def compiler_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is + respecively less-than, equal-to, or greater-than compiler b of pkgname. + One compiler is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'compiler', a, b, False, None) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3b5d16c7a75..83b1416e36d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1653,6 +1653,40 @@ def dep_string(self): return ''.join("^" + dep.format() for dep in self.sorted_deps()) + def __cmp__(self, other): + #Package name sort order is not configurable, always goes alphabetical + if self.name != other.name: + return cmp(self.name, other.name) + + #Package version is second in compare order + pkgname = self.name + if self.versions != other.versions: + return spack.pkgsort.version_compare(pkgname, + self.versions, other.versions) + + #Compiler is third + if self.compiler != other.compiler: + return spack.pkgsort.compiler_compare(pkgname, + self.compiler, other.compiler) + + #Variants + if self.variants != other.variants: + return spack.pkgsort.variant_compare(pkgname, + self.variants, other.variants) + + #Architecture + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) + + #Dependency is not configurable + if self.dep_hash() != other.dep_hash(): + return -1 if self.dep_hash() < other.dep_hash() else 1 + + #Equal specs + return 0 + + def __str__(self): return self.format() + self.dep_string() From 8d7b7e5d5dadcec9b997b94d95898a4134e122b2 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 12:20:32 -0700 Subject: [PATCH 004/339] Use preferred package rules when concretize'ing specs --- lib/spack/spack/concretize.py | 68 +++++++++++++++++++---------------- lib/spack/spack/spec.py | 2 +- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb8..0f258c90968 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -38,6 +38,7 @@ import spack.architecture import spack.error from spack.version import * +from functools import partial @@ -49,8 +50,8 @@ class DefaultConcretizer(object): def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take - the most recent available version, and default to the package's - version if there are no avaialble versions. + the preferred version from spackconfig, and default to the package's + version if there are no available versions. TODO: In many cases we probably want to look for installed versions of each package and use an installed version @@ -68,12 +69,14 @@ def concretize_version(self, spec): # If there are known available versions, return the most recent # version that satisfies the spec pkg = spec.package + cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions - if any(v.satisfies(sv) for sv in spec.versions)]) + if any(v.satisfies(sv) for sv in spec.versions)], + cmp=cmp_versions) if valid_versions: - spec.versions = ver([valid_versions[-1]]) + spec.versions = ver([valid_versions[0]]) else: # We don't know of any SAFE versions that match the given # spec. Grab the spec's versions and grab the highest @@ -138,10 +141,10 @@ def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not - concrete, then give it a valid version. If there is no - ancestor with a compiler, use the system default compiler. + concrete, then used the preferred compiler as specified in + spackconfig. - Intuition: Use the system default if no package that depends on + Intuition: Use the spackconfig default if no package that depends on this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. @@ -153,40 +156,43 @@ def concretize_compiler(self, spec): spec.compiler in all_compilers): return False - try: - nearest = next(p for p in spec.traverse(direction='parents') - if p.compiler is not None).compiler - - if not nearest in all_compilers: - # Take the newest compiler that saisfies the spec - matches = sorted(spack.compilers.find(nearest)) - if not matches: - raise UnavailableCompilerVersionError(nearest) - - # copy concrete version into nearest spec - nearest.versions = matches[-1].versions.copy() - assert(nearest.concrete) - - spec.compiler = nearest.copy() - - except StopIteration: - spec.compiler = spack.compilers.default_compiler().copy() - + # Find the parent spec that has a compiler, or the root if none do + parent_spec = next(p for p in spec.traverse(direction='parents') + if p.compiler is not None or not p.dependents) + parent_compiler = parent_spec.compiler + assert(parent_spec) + + # Check if the compiler is already fully specified + if parent_compiler in all_compilers: + spec.compiler = parent_compiler.copy() + return True + + # Filter the compilers into a sorted list based on the compiler_order from spackconfig + compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler) + cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name) + matches = sorted(compiler_list, cmp=cmp_compilers) + if not matches: + raise UnavailableCompilerVersionError(parent_compiler) + + # copy concrete version into parent_compiler + spec.compiler = matches[0].copy() + assert(spec.compiler.concrete) return True # things changed. - def choose_provider(self, spec, providers): + def choose_provider(self, package_spec, spec, providers): """This is invoked for virtual specs. Given a spec with a virtual name, say "mpi", and a list of specs of possible providers of that spec, select a provider and return it. """ assert(spec.virtual) assert(providers) + + provider_cmp = partial(spack.pkgsort.provider_compare, package_spec.name, spec.name) + sorted_providers = sorted(providers, cmp=provider_cmp) + first_key = sorted_providers[0] - index = spack.spec.index_specs(providers) - first_key = sorted(index.keys())[0] - latest_version = sorted(index[first_key])[-1] - return latest_version + return first_key class UnavailableCompilerVersionError(spack.error.SpackError): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 83b1416e36d..41496b0e9d5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -796,7 +796,7 @@ def _expand_virtual_packages(self): for spec in virtuals: providers = spack.db.providers_for(spec) - concrete = spack.concretizer.choose_provider(spec, providers) + concrete = spack.concretizer.choose_provider(self, spec, providers) concrete = concrete.copy() spec._replace_with(concrete) changed = True From ee68a76a193890231d6df7fa7934d42e3708540b Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 29 May 2015 14:57:24 -0700 Subject: [PATCH 005/339] Bug fixes from testing spack preferred packages --- lib/spack/spack/preferred_packages.py | 7 +++---- var/spack/mock_packages/mpich/package.py | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 248508fe809..bc5271f693d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -112,9 +112,8 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon elif a_index == None and b_index != None: return 1 elif a_index != None and b_index == a_index: return -1 * cmp(a, b) elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) - elif a < b: return 1 * reverse - elif b < a: return -1 * reverse - else: return 0 + else: return cmp(a, b) * reverse + # Given a sort order specified by the pkgname/component/second_key, return @@ -148,7 +147,7 @@ def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. One version is less-than another if it is preferred over the other.""" - return self._spec_compare(pkgname, 'version', a, b, False, None) + return self._spec_compare(pkgname, 'version', a, b, True, None) def variant_compare(self, pkgname, a, b): diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py index f77d3efc5d1..e4110ad5301 100644 --- a/var/spack/mock_packages/mpich/package.py +++ b/var/spack/mock_packages/mpich/package.py @@ -38,6 +38,7 @@ class Mpich(Package): version('3.0.2', 'foobarbaz') version('3.0.1', 'foobarbaz') version('3.0', 'foobarbaz') + version('1.0', 'foobarbas') provides('mpi@:3', when='@3:') provides('mpi@:1', when='@:1') From 987cd9e78f99aa6ee6f3a48a4b8f556a68cb2965 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:27:13 -0700 Subject: [PATCH 006/339] Update docs for YAML configuration files and preferred concretization --- lib/spack/docs/basic_usage.rst | 37 +++++++----- lib/spack/docs/mirrors.rst | 9 ++- lib/spack/docs/packaging_guide.rst | 66 ++++++++++++++++++++- lib/spack/docs/site_configuration.rst | 83 --------------------------- 4 files changed, 91 insertions(+), 104 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db6..5d5438220cc 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to where the compiler is installed. For example:: $ spack compiler add /usr/local/tools/ic-13.0.079 - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml intel@13.0.079 Or you can run ``spack compiler add`` with no arguments to force @@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your $ module load gcc-4.9.0 $ spack compiler add - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml gcc@4.9.0 This loads the environment module for gcc-4.9.0 to get it into the @@ -398,27 +398,34 @@ Manual compiler configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If auto-detection fails, you can manually configure a compiler by -editing your ``~/.spackconfig`` file. You can do this by running -``spack config edit``, which will open the file in your ``$EDITOR``. +editing your ``~/.spack/compilers.yaml`` file. You can do this by running +``spack config edit compilers``, which will open the file in your ``$EDITOR``. Each compiler configuration in the file looks like this:: ... - [compiler "intel@15.0.0"] - cc = /usr/local/bin/icc-15.0.024-beta - cxx = /usr/local/bin/icpc-15.0.024-beta - f77 = /usr/local/bin/ifort-15.0.024-beta - fc = /usr/local/bin/ifort-15.0.024-beta - ... + chaos_5_x86_64_ib: + ... + intel@15.0.0: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + ... + +The chaos_5_x86_64_ib string is an architecture string, and multiple +compilers can be listed underneath an architecture. The architecture +string may be replaced with the string 'all' to signify compilers that +work on all architectures. For compilers, like ``clang``, that do not support Fortran, put ``None`` for ``f77`` and ``fc``:: - [compiler "clang@3.3svn"] - cc = /usr/bin/clang - cxx = /usr/bin/clang++ - f77 = None - fc = None + clang@3.3svn: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: None + fc: None Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index d732a3dd54f..7581a0e9ede 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name:: Mirror precedence ---------------------------- -Adding a mirror really just adds a section in ``~/.spackconfig``:: +Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``:: - [mirror "local_filesystem"] - url = file:///Users/gamblin2/spack-mirror-2014-06-24 - [mirror "remote_server"] - url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + mirrors: + - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 + - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..5094f739c45 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -632,7 +632,7 @@ Default revision instead. Revisions - Add ``hg`` and ``revision``parameters: + Add ``hg`` and ``revision`` parameters: .. code-block:: python @@ -1524,6 +1524,70 @@ This is useful when you want to know exactly what Spack will do when you ask for a particular spec. +``Concretization Policies`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A user may have certain perferrences for how packages should +be concretized on their system. For example, one user may prefer packages +built with OpenMPI and the Intel compiler. Another user may prefer +packages be built with MVAPICH and GCC. + +Spack's ``preferred`` configuration can be used to set defaults for sites or users. +Spack uses this configuration to make decisions about which compilers, package +versions, depends_on, and variants it should prefer during concretization. + +The preferred configuration can be controlled by editing the +``~/.spack/preferred.yaml`` file for user configuations, or the + + +Here's an example preferred.yaml file: + +.. code-block:: sh + + preferred: + dyninst: + compiler: gcc@4.9 + variants: +debug + gperftools: + version: 2.2, 2.4, 2.3 + all: + compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi + providers: + mpi: mvapich, mpich, openmpi + +At a high level, this example is specifying how packages should be +concretized. The dyninst package should prefer using gcc 4.9 and +be built with debug options. The gperftools package should prefer version +2.2 over 2.4. Every package on the system should prefer mvapich for +its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten +on the command line if explicitly requested. + +Each preferred.yaml file begin with the string ``preferred:`` and +each subsequent entry is indented underneath it. The next layer contains +package names or the special string ``all`` (which applies to +every package). Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec +``constraints``, with earlier entries in the list being prefered over +latter entries. + +Sometimes a package installation may have constraints that forbid +the first concretization rule, in which case Spack will use the first +legal concretization rule. Going back to the example, if a user +requests gperftools 2.3 or latter, then Spack will install version 2.4 +as the 2.4 version of gperftools is preferred over 2.3. + +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, +xlc isn't listed in the compiler list. Every listed compiler from +gcc to pgi will thus be preferred over the xlc compiler. + +The syntax for the ``providers`` section differs slightly from other +concretization rules. A provider lists a value that packages may +``depend_on`` (e.g, mpi) and a list of rules for fulfilling that +dependency. + .. _install-method: Implementing the ``install`` method diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index b03df29573b..1e6740a434f 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -54,89 +54,6 @@ more elements to the list to indicate where your own site's temporary directory is. -.. _concretization-policies: - -Concretization policies ----------------------------- - -When a user asks for a package like ``mpileaks`` to be installed, -Spack has to make decisions like what version should be installed, -what compiler to use, and how its dependencies should be configured. -This process is called *concretization*, and it's covered in detail in -:ref:`its own section `. - -The default concretization policies are in the -:py:mod:`spack.concretize` module, specifically in the -:py:class:`spack.concretize.DefaultConcretizer` class. These are the -important methods used in the concretization process: - -* :py:meth:`concretize_version(self, spec) ` -* :py:meth:`concretize_architecture(self, spec) ` -* :py:meth:`concretize_compiler(self, spec) ` -* :py:meth:`choose_provider(self, spec, providers) ` - -The first three take a :py:class:`Spec ` object and -modify it by adding constraints for the version. For example, if the -input spec had a version range like `1.0:5.0.3`, then the -``concretize_version`` method should set the spec's version to a -*single* version in that range. Likewise, ``concretize_architecture`` -selects an architecture when the input spec does not have one, and -``concretize_compiler`` needs to set both a concrete compiler and a -concrete compiler version. - -``choose_provider()`` affects how concrete implementations are chosen -based on a virtual dependency spec. The input spec is some virtual -dependency and the ``providers`` index is a :py:class:`ProviderIndex -` object. The ``ProviderIndex`` maps -the virtual spec to specs for possible implementations, and -``choose_provider()`` should simply choose one of these. The -``concretize_*`` methods will be called on the chosen implementation -later, so there is no need to fully concretize the spec when returning -it. - -The ``DefaultConcretizer`` is intended to provide sensible defaults -for each policy, but there are certain choices that it can't know -about. For example, one site might prefer ``OpenMPI`` over ``MPICH``, -or another might prefer an old version of some packages. These types -of special cases can be integrated with custom concretizers. - -Writing a custom concretizer -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To write your own concretizer, you need only subclass -``DefaultConcretizer`` and override the methods you want to change. -For example, you might write a class like this to change *only* the -``concretize_version()`` behavior: - -.. code-block:: python - - from spack.concretize import DefaultConcretizer - - class MyConcretizer(DefaultConcretizer): - def concretize_version(self, spec): - # implement custom logic here. - -Once you have written your custom concretizer, you can make Spack use -it by editing ``globals.py``. Find this part of the file: - -.. code-block:: python - - # - # This controls how things are concretized in spack. - # Replace it with a subclass if you want different - # policies. - # - concretizer = DefaultConcretizer() - -Set concretizer to *your own* class instead of the default: - -.. code-block:: python - - concretizer = MyConcretizer() - -The next time you run Spack, your changes should take effect. - - Profiling ~~~~~~~~~~~~~~~~~~~~~ From 53cde110b1d25f8fcf89d7c26ef02bb073cb6a29 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:27:46 -0700 Subject: [PATCH 007/339] Update Spack mirror command to match docs --- lib/spack/spack/cmd/mirror.py | 12 ++++++------ lib/spack/spack/stage.py | 5 ++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9ac..baf64d30fc8 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -75,8 +75,8 @@ def mirror_add(args): if url.startswith('/'): url = 'file://' + url - mirror_dict = { args.name : url } - spack.config.add_to_mirror_config({ args.name : url }) + newmirror = [ { args.name : url } ] + spack.config.add_to_mirror_config(newmirror) def mirror_remove(args): @@ -90,15 +90,15 @@ def mirror_remove(args): def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_mirror_config() - if not sec_names: + mirrors = spack.config.get_mirror_config() + if not mirrors: tty.msg("No mirrors configured.") return - max_len = max(len(s) for s in sec_names) + max_len = max(len(name) for name,path in mirrors) fmt = "%%-%ds%%s" % (max_len + 4) - for name, val in sec_names.iteritems(): + for name, val in mirrors: print fmt % (name, val) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 008c5f04298..c70c7a84a4e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,6 +26,7 @@ import re import shutil import tempfile +import sys import llnl.util.tty as tty from llnl.util.filesystem import * @@ -344,9 +345,7 @@ def destroy(self): def _get_mirrors(): """Get mirrors from spack configuration.""" - config = spack.config.get_mirror_config() - return [val for name, val in config.iteritems()] - + return [path for name, path in spack.config.get_mirror_config()] def ensure_access(file=spack.stage_path): From 53d70fff0121a05fb21c02363570f81573bbeffa Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Tue, 7 Jul 2015 16:28:51 -0700 Subject: [PATCH 008/339] Fix type error with YAML config when merging lists from different configs. --- lib/spack/spack/config.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dbe225960a8..f3526b19fa3 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -98,6 +98,7 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp +import copy _config_sections = {} class _ConfigCategory: @@ -159,21 +160,19 @@ def _merge_dicts(d1, d2): if not d2: return d1 - for key2, val2 in d2.iteritems(): - if not key2 in d1: - d1[key2] = val2 - continue - val1 = d1[key2] - if isinstance(val1, dict) and isinstance(val2, dict): - d1[key2] = _merge_dicts(val1, val2) - continue - if isinstance(val1, list) and isinstance(val2, list): - val1.extend(val2) - seen = set() - d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ] - continue - d1[key2] = val2 - return d1 + if (type(d1) is list) and (type(d2) is list): + d1.extend(d2) + return d1 + + if (type(d1) is dict) and (type(d2) is dict): + for key2, val2 in d2.iteritems(): + if not key2 in d1: + d1[key2] = val2 + else: + d1[key2] = _merge_dicts(d1[key2], val2) + return d1 + + return d2 def get_config(category_name): @@ -225,8 +224,8 @@ def get_compilers_config(arch=None): def get_mirror_config(): - """Get the mirror configuration from config files""" - return get_config('mirrors') + """Get the mirror configuration from config files as a list of name/location tuples""" + return [x.items()[0] for x in get_config('mirrors')] def get_preferred_config(): From 650c9d4e36c6a58cf6bca0e6abd580ee54d8e175 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 16 Sep 2015 10:56:11 -0700 Subject: [PATCH 009/339] Allow spack to build against external non-spack-installed packages. --- lib/spack/spack/__init__.py | 6 ++ lib/spack/spack/abi.py | 128 ++++++++++++++++++++++ lib/spack/spack/concretize.py | 140 ++++++++++++++++++++++--- lib/spack/spack/config.py | 59 ++++++++++- lib/spack/spack/directory_layout.py | 8 ++ lib/spack/spack/package.py | 3 + lib/spack/spack/preferred_packages.py | 10 +- lib/spack/spack/spec.py | 58 +++++----- var/spack/packages/mpich/package.py | 1 + var/spack/packages/mvapich2/package.py | 17 ++- 10 files changed, 387 insertions(+), 43 deletions(-) create mode 100644 lib/spack/spack/abi.py diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index bd8478fb98e..5783005b5b8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -77,6 +77,12 @@ from spack.preferred_packages import PreferredPackages pkgsort = PreferredPackages() +# +# This tests ABI compatibility between packages +# +from spack.abi import ABI +abi = ABI() + # # This controls how things are concretized in spack. # Replace it with a subclass if you want different diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py new file mode 100644 index 00000000000..f0a997703cd --- /dev/null +++ b/lib/spack/spack/abi.py @@ -0,0 +1,128 @@ +############################################################################## +# Copyright (c) 2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import spack +import spack.spec +from spack.spec import CompilerSpec +from spack.util.executable import Executable, ProcessError +from llnl.util.lang import memoized + +class ABI(object): + """This class provides methods to test ABI compatibility between specs. + The current implementation is rather rough and could be improved.""" + + def architecture_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible architectures.""" + return not parent.architecture or not child.architecture or parent.architecture == child.architecture + + + @memoized + def _gcc_get_libstdcxx_version(self, version): + """Returns gcc ABI compatibility info by getting the library version of + a compiler's libstdc++.so or libgcc_s.so""" + spec = CompilerSpec("gcc", version) + compilers = spack.compilers.compilers_for_spec(spec) + if not compilers: + return None + compiler = compilers[0] + rungcc = None + libname = None + output = None + if compiler.cxx: + rungcc = Executable(compiler.cxx) + libname = "libstdc++.so" + elif compiler.cc: + rungcc = Executable(compiler.cc) + libname = "libgcc_s.so" + else: + return None + try: + output = rungcc("--print-file-name=%s" % libname, return_output=True) + except ProcessError, e: + return None + if not output: + return None + libpath = os.readlink(output.strip()) + if not libpath: + return None + return os.path.basename(libpath) + + + @memoized + def _gcc_compiler_compare(self, pversion, cversion): + """Returns true iff the gcc version pversion and cversion + are ABI compatible.""" + plib = self._gcc_get_libstdcxx_version(pversion) + clib = self._gcc_get_libstdcxx_version(cversion) + if not plib or not clib: + return False + return plib == clib + + + def _intel_compiler_compare(self, pversion, cversion): + """Returns true iff the intel version pversion and cversion + are ABI compatible""" + + #Test major and minor versions. Ignore build version. + if (len(pversion.version) < 2 or len(cversion.version) < 2): + return False + return (pversion.version[0] == cversion.version[0]) and \ + (pversion.version[1] == cversion.version[1]) + + + def compiler_compatible(self, parent, child, **kwargs): + """Returns true iff the compilers for parent and child specs are ABI compatible""" + if not parent.compiler or not child.compiler: + return True + + if parent.compiler.name != child.compiler.name: + #Different compiler families are assumed ABI incompatible + return False + + if kwargs.get('loose', False): + return True + + for pversion in parent.compiler.versions: + for cversion in child.compiler.versions: + #For a few compilers use specialized comparisons. Otherwise + # match on version match. + if pversion.satisfies(cversion): + return True + elif parent.compiler.name == "gcc" and \ + self._gcc_compiler_compare(pversion, cversion): + return True + elif parent.compiler.name == "intel" and \ + self._intel_compiler_compare(pversion, cversion): + return True + return False + + + def compatible(self, parent, child, **kwargs): + """Returns true iff a parent and child spec are ABI compatible""" + loosematch = kwargs.get('loose', False) + return self.architecture_compatible(parent, child) and \ + self.compiler_compatible(parent, child, loose=loosematch) + diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 0f258c90968..01ff163493b 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,13 +33,16 @@ TODO: make this customizable and allow users to configure concretization policies. """ +import spack import spack.spec import spack.compilers import spack.architecture import spack.error from spack.version import * from functools import partial - +from spec import DependencyMap +from itertools import chain +from spack.config import * class DefaultConcretizer(object): @@ -48,6 +51,107 @@ class DefaultConcretizer(object): default concretization strategies, or you can override all of them. """ + def _find_other_spec(self, spec, condition): + """Searches the dag from spec in an intelligent order and looks + for a spec that matches a condition""" + dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) + found = next((x for x in dagiter if x is not spec and condition(x)), None) + if found: + return found + dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) + searched = list(dagiter) + found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None) + if found: + return found + if condition(spec): + return spec + return None + + + def _valid_virtuals_and_externals(self, spec): + """Returns a list of spec/external-path pairs for both virtuals and externals + that can concretize this spec.""" + + # Get a list of candidate packages that could satisfy this spec + packages = [] + if spec.virtual: + providers = spack.db.providers_for(spec) + if not providers: + raise UnsatisfiableProviderSpecError(providers[0], spec) + spec_w_preferred_providers = self._find_other_spec(spec, \ + lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + if not spec_w_preferred_providers: + spec_w_preferred_providers = spec + provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) + packages = sorted(providers, cmp=provider_cmp) + else: + if not spec_externals(spec) or spec.external: + return None + packages = [spec] + + # For each candidate package, if it has externals add those to the candidates + # if it's a nobuild, then only add the externals. + result = [] + all_compilers = spack.compilers.all_compilers() + for pkg in packages: + externals = spec_externals(pkg) + buildable = not is_spec_nobuild(pkg) + if buildable: + result.append((pkg, None)) + if externals: + sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) + for external in sorted_externals: + if external[0].satisfies(spec): + result.append(external) + if not result: + raise NoBuildError(spec) + return result + + + def concretize_virtual_and_external(self, spec): + """From a list of candidate virtual and external packages, concretize to one that + is ABI compatible with the rest of the DAG.""" + candidates = self._valid_virtuals_and_externals(spec) + if not candidates: + return False + + #Find the another spec in the dag that has a compiler. We'll use that + # spec to test compiler compatibility. + other_spec = self._find_other_spec(spec, lambda(x): x.compiler) + if not other_spec: + other_spec = spec.root + + #Choose an ABI-compatible candidate, or the first match otherwise. + candidate = None + if other_spec: + candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) + if not candidate: + #Try a looser ABI matching + candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) + if not candidate: + #Pick the first choice + candidate = candidates[0] + external = candidate[1] + candidate_spec = candidate[0] + + #Refine this spec to the candidate. + changed = False + if spec.virtual: + spec._replace_with(candidate_spec) + changed = True + if spec._dup(candidate_spec, deps=False, cleardeps=False): + changed = True + if not spec.external and external: + spec.external = external + changed = True + #If we're external then trim the dependencies + if external and spec.dependencies: + changed = True + spec.depencencies = DependencyMap() + + return changed + + def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take the preferred version from spackconfig, and default to the package's @@ -150,31 +254,32 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() - + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return False - # Find the parent spec that has a compiler, or the root if none do - parent_spec = next(p for p in spec.traverse(direction='parents') - if p.compiler is not None or not p.dependents) - parent_compiler = parent_spec.compiler - assert(parent_spec) + #Find the another spec that has a compiler, or the root if none do + other_spec = self._find_other_spec(spec, lambda(x) : x.compiler) + if not other_spec: + other_spec = spec.root + other_compiler = other_spec.compiler + assert(other_spec) # Check if the compiler is already fully specified - if parent_compiler in all_compilers: - spec.compiler = parent_compiler.copy() + if other_compiler in all_compilers: + spec.compiler = other_compiler.copy() return True # Filter the compilers into a sorted list based on the compiler_order from spackconfig - compiler_list = all_compilers if not parent_compiler else spack.compilers.find(parent_compiler) - cmp_compilers = partial(spack.pkgsort.compiler_compare, parent_spec.name) + compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) + cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: - raise UnavailableCompilerVersionError(parent_compiler) + raise UnavailableCompilerVersionError(other_compiler) - # copy concrete version into parent_compiler + # copy concrete version into other_compiler spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. @@ -210,3 +315,12 @@ class NoValidVersionError(spack.error.SpackError): def __init__(self, spec): super(NoValidVersionError, self).__init__( "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) + + +class NoBuildError(spack.error.SpackError): + """Raised when a package is configured with the nobuild option, but + no satisfactory external versions can be found""" + def __init__(self, spec): + super(NoBuildError, self).__init__( + "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) + diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f3526b19fa3..60577c45b36 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -90,9 +90,12 @@ import exceptions import sys import copy - -from llnl.util.lang import memoized +import inspect +import glob +import imp +import spack.spec import spack.error +from llnl.util.lang import memoized from external import yaml from external.yaml.error import MarkedYAMLError @@ -116,6 +119,9 @@ def __init__(self, n, f, m): _ConfigCategory('compilers', 'compilers.yaml', True) _ConfigCategory('mirrors', 'mirrors.yaml', True) _ConfigCategory('preferred', 'preferred.yaml', True) +_ConfigCategory('view', 'views.yaml', True) +_ConfigCategory('preferred', 'preferred.yaml', True) +_ConfigCategory('packages', 'packages.yaml', True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), @@ -233,6 +239,55 @@ def get_preferred_config(): return get_config('preferred') +@memoized +def get_packages_config(): + """Get the externals configuration from config files""" + package_config = get_config('packages') + if not package_config: + return {} + indexed_packages = {} + for p in package_config: + package_name = spack.spec.Spec(p.keys()[0]).name + if package_name not in indexed_packages: + indexed_packages[package_name] = [] + indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() }) + return indexed_packages + + +def is_spec_nobuild(spec): + """Return true if the spec pkgspec is configured as nobuild""" + allpkgs = get_packages_config() + name = spec.name + if not name in allpkgs: + return False + for itm in allpkgs[name]: + for pkg,conf in itm.iteritems(): + if pkg.satisfies(spec): + if conf.get('nobuild', False): + return True + return False + + +def spec_externals(spec): + """Return a list of spec, directory pairs for each external location for spec""" + allpkgs = get_packages_config() + name = spec.name + spec_locations = [] + + if not name in allpkgs: + return [] + for itm in allpkgs[name]: + for pkg,conf in itm.iteritems(): + if not pkg.satisfies(spec): + continue + path = conf.get('path', None) + if not path: + continue + spec_locations.append( (pkg, path) ) + return spec_locations + + + def get_config_scope_dirname(scope): """For a scope return the config directory""" global config_scopes diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 85ecc1ce2b1..83e6eb566a7 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -187,6 +187,14 @@ def hidden_file_paths(self): def relative_path_for_spec(self, spec): _check_concrete(spec) + + if spec.external: + return spec.external + + enabled_variants = ( + '-' + v.name for v in spec.variants.values() + if v.enabled) + dir_name = "%s-%s-%s" % ( spec.name, spec.version, diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d05905..1e2f0378c8e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -752,6 +752,9 @@ def do_install(self, if not self.spec.concrete: raise ValueError("Can only install concrete packages.") + if self.spec.external: + return + if os.path.exists(self.prefix): tty.msg("%s is already installed in %s." % (self.name, self.prefix)) return diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index bc5271f693d..bc2a4ac234d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -35,9 +35,10 @@ def __init__(self): #Given a package name, sort component (e.g, version, compiler, ...), and # a second_key (used by providers), return the list - def _order_for_package(self, pkgname, component, second_key): + def _order_for_package(self, pkgname, component, second_key, test_all=True): pkglist = [pkgname] - pkglist.append('all') + if test_all: + pkglist.append('all') for pkg in pkglist: if not pkg in self.preferred: continue @@ -143,6 +144,11 @@ def provider_compare(self, pkgname, provider_str, a, b): return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) + def spec_has_preferred_provider(self, pkgname, provider_str): + """Return True iff the named package has a list of preferred provider""" + return bool(self._order_for_package(pkgname, 'providers', provider_str, False)) + + def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respecively less-than, equal-to, or greater-than version b of pkgname. diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 41496b0e9d5..6984b4a174a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -419,6 +419,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): # package.py files for. self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) + self.external = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -426,7 +427,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): for dep in dep_like: spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - + # # Private routines here are called by the parser when building a spec. @@ -751,12 +752,11 @@ def _concretize_helper(self, presets=None, visited=None): # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. - if not self.virtual: - changed |= any( - (spack.concretizer.concretize_architecture(self), - spack.concretizer.concretize_compiler(self), - spack.concretizer.concretize_version(self), - spack.concretizer.concretize_variants(self))) + changed |= any( + (spack.concretizer.concretize_architecture(self), + spack.concretizer.concretize_compiler(self), + spack.concretizer.concretize_version(self), + spack.concretizer.concretize_variants(self))) presets[self.name] = self visited.add(self.name) @@ -789,21 +789,18 @@ def _expand_virtual_packages(self): a problem. """ changed = False - while True: - virtuals =[v for v in self.traverse() if v.virtual] - if not virtuals: - return changed + done = False + while not done: + done = True + for spec in list(self.traverse()): + if spack.concretizer.concretize_virtual_and_external(spec): + done = False + changed = True - for spec in virtuals: - providers = spack.db.providers_for(spec) - concrete = spack.concretizer.choose_provider(self, spec, providers) - concrete = concrete.copy() - spec._replace_with(concrete) - changed = True - - # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merge constraints. - changed |= self.normalize(force=True) + # If there are duplicate providers or duplicate provider deps, this + # consolidates them and merge constraints. + changed |= self.normalize(force=True) + return changed def concretize(self): @@ -830,7 +827,6 @@ def concretize(self): self._concretize_helper()) changed = any(changes) force=True - self._concrete = True @@ -1346,15 +1342,26 @@ def _dup(self, other, **kwargs): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ + + # We don't count dependencies as changes here + changed = True + if hasattr(self, 'name'): + changed = (self.name != other.name and self.versions != other.versions and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ + self.variants != other.variants and self._normal != other._normal and \ + self.concrete != other.concrete and self.external != other.external) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None - self.dependents = DependencyMap() - self.dependencies = DependencyMap() + if kwargs.get('cleardeps', True): + self.dependents = DependencyMap() + self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self + self.external = other.external # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1372,6 +1379,8 @@ def _dup(self, other, **kwargs): # Since we preserved structure, we can copy _normal safely. self._normal = other._normal self._concrete = other._concrete + self.external = other.external + return changed def copy(self, **kwargs): @@ -1796,6 +1805,7 @@ def spec(self): spec.variants = VariantMap(spec) spec.architecture = None spec.compiler = None + spec.external = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index b6b2dfde214..dfff22152da 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -45,6 +45,7 @@ def setup_dependent_environment(self, module, spec, dep_spec): os.environ['MPICH_F77'] = 'f77' os.environ['MPICH_F90'] = 'f90' + module.mpicc = join_path(self.prefix.bin, 'mpicc') def install(self, spec, prefix): config_args = ["--prefix=" + prefix, diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index ca0b1287c1c..93bce011b71 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -11,10 +11,17 @@ class Mvapich2(Package): version('2.0', '9fbb68a4111a8b6338e476dc657388b4', url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') + + version('2.1', '0095ceecb19bbb7fb262131cb9c2cdd6', + url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.1.tar.gz') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 + variant('psm', default=False, description="build with psm") + + variant('pmi', default=False, description="build with pmi") + depends_on('pmgr_collective', when='+pmi') def install(self, spec, prefix): # we'll set different configure flags depending on our environment @@ -80,7 +87,13 @@ def install(self, spec, prefix): configure_args.append("--with-device=ch3:psm") else: # throw this flag on IB systems - configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") + configure_args.append("--with-device=ch3:mrail") + configure_args.append("--with-rdma=gen2") + + if "+pmi" in spec: + configure_args.append("--with-pmi=pmgr_collective" % spec['pmgr_collective'].prefix) + else: + configure_args.append("--with-pmi=slurm") # TODO: shared-memory build @@ -93,7 +106,7 @@ def install(self, spec, prefix): "--enable-f77", "--enable-fc", "--enable-cxx", "--enable-shared", "--enable-sharedlibs=gcc", "--enable-debuginfo", - "--with-pm=no", "--with-pmi=slurm", + "--with-pm=no", "--enable-romio", "--with-file-system=lustre+nfs+ufs", "--disable-mpe", "--without-mpe", "--disable-silent-rules", From e4d2ba30b57618da388a1a990381d149b33d7aba Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 2 Oct 2015 11:00:41 -0700 Subject: [PATCH 010/339] Fix failure in spack.test.config.ConfigTest from incorrect compiler config merging --- lib/spack/spack/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 60577c45b36..712a2b78fc0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -174,8 +174,12 @@ def _merge_dicts(d1, d2): for key2, val2 in d2.iteritems(): if not key2 in d1: d1[key2] = val2 - else: + elif type(d1[key2]) is dict and type(val2) is dict: d1[key2] = _merge_dicts(d1[key2], val2) + elif (type(d1) is list) and (type(d2) is list): + d1.extend(d2) + else: + d1[key2] = val2 return d1 return d2 @@ -360,7 +364,7 @@ def add_to_mirror_config(addition_dict, scope=None): def add_to_compiler_config(addition_dict, scope=None, arch=None): - """Add compilerss to the configuration files""" + """Add compilers to the configuration files""" if not arch: arch = spack.architecture.sys_type() add_to_config('compilers', { arch : addition_dict }, scope) From 18f0b24a7f21ec7b46510f45867386b7600bbc55 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 5 Oct 2015 11:30:48 -0700 Subject: [PATCH 011/339] Add tests for spack external dependencies, plus fixes for issues found by those tests. --- lib/spack/spack/concretize.py | 9 +++-- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/concretize.py | 28 ++++++++++++++ .../site_spackconfig/packages.yaml | 13 +++++++ .../mock_packages/externalprereq/package.py | 34 +++++++++++++++++ .../mock_packages/externaltest/package.py | 37 +++++++++++++++++++ .../mock_packages/externaltool/package.py | 36 ++++++++++++++++++ .../mock_packages/externalvirtual/package.py | 37 +++++++++++++++++++ 8 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 var/spack/mock_configs/site_spackconfig/packages.yaml create mode 100644 var/spack/mock_packages/externalprereq/package.py create mode 100644 var/spack/mock_packages/externaltest/package.py create mode 100644 var/spack/mock_packages/externaltool/package.py create mode 100644 var/spack/mock_packages/externalvirtual/package.py diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 01ff163493b..c27a023136a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -85,8 +85,8 @@ def _valid_virtuals_and_externals(self, spec): provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) packages = sorted(providers, cmp=provider_cmp) else: - if not spec_externals(spec) or spec.external: - return None + if spec.external: + return False packages = [spec] # For each candidate package, if it has externals add those to the candidates @@ -129,7 +129,7 @@ def concretize_virtual_and_external(self, spec): #Try a looser ABI matching candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) if not candidate: - #Pick the first choice + #No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] external = candidate[1] candidate_spec = candidate[0] @@ -144,10 +144,11 @@ def concretize_virtual_and_external(self, spec): if not spec.external and external: spec.external = external changed = True + #If we're external then trim the dependencies if external and spec.dependencies: changed = True - spec.depencencies = DependencyMap() + spec.dependencies = DependencyMap() return changed diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6984b4a174a..49b67cd3614 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1022,7 +1022,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual: + if self.virtual or self.external: return False # Combine constraints from package deps with constraints from diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a1..f81a2f5af81 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -192,3 +192,31 @@ def test_compiler_inheritance(self): # TODO: not exactly the syntax I would like. self.assertTrue(spec['libdwarf'].compiler.satisfies('clang')) self.assertTrue(spec['libelf'].compiler.satisfies('clang')) + + + def test_external_package(self): + spec = Spec('externaltool') + spec.concretize() + + self.assertEqual(spec['externaltool'].external, '/path/to/external_tool') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + + + def test_nobuild_package(self): + got_error = False + spec = Spec('externaltool%clang') + try: + spec.concretize() + except spack.concretize.NoBuildError: + got_error = True + self.assertTrue(got_error) + + + def test_external_and_virtual(self): + spec = Spec('externaltest') + spec.concretize() + self.assertTrue(spec['externaltool'].external, '/path/to/external_tool') + self.assertTrue(spec['stuff'].external, '/path/to/external_virtual_gcc') + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + self.assertTrue(spec['stuff'].compiler.satisfies('gcc')) diff --git a/var/spack/mock_configs/site_spackconfig/packages.yaml b/var/spack/mock_configs/site_spackconfig/packages.yaml new file mode 100644 index 00000000000..eb52c6cf113 --- /dev/null +++ b/var/spack/mock_configs/site_spackconfig/packages.yaml @@ -0,0 +1,13 @@ +packages: + - externaltool: + nobuild: True + - externaltool@1.0%gcc@4.5.0: + path: /path/to/external_tool + - externalvirtual@2.0%clang@3.3: + path: /path/to/external_virtual_clang + nobuild: True + - externalvirtual@1.0%gcc@4.5.0: + path: /path/to/external_virtual_gcc + nobuild: True + + diff --git a/var/spack/mock_packages/externalprereq/package.py b/var/spack/mock_packages/externalprereq/package.py new file mode 100644 index 00000000000..7d639256939 --- /dev/null +++ b/var/spack/mock_packages/externalprereq/package.py @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalprereq(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/prereq-1.0.tar.gz" + + version('1.4', 'f1234567890abcdef1234567890abcde') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externaltest/package.py b/var/spack/mock_packages/externaltest/package.py new file mode 100644 index 00000000000..c546922f876 --- /dev/null +++ b/var/spack/mock_packages/externaltest/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltest(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/test-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('stuff') + depends_on('externaltool') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externaltool/package.py b/var/spack/mock_packages/externaltool/package.py new file mode 100644 index 00000000000..af902bd70eb --- /dev/null +++ b/var/spack/mock_packages/externaltool/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltool(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/tool-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/externalvirtual/package.py b/var/spack/mock_packages/externalvirtual/package.py new file mode 100644 index 00000000000..722c1e1c532 --- /dev/null +++ b/var/spack/mock_packages/externalvirtual/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalvirtual(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/stuff-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + version('2.0', '234567890abcdef1234567890abcdef1') + + provides('stuff') + + def install(self, spec, prefix): + pass From fac4428766fb0a6b6cd357b654215f55df1220d4 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 5 Oct 2015 14:04:33 -0700 Subject: [PATCH 012/339] Documentation for external packages. --- lib/spack/docs/site_configuration.rst | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 1e6740a434f..a7211a9d95b 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -54,6 +54,79 @@ more elements to the list to indicate where your own site's temporary directory is. +External Packages +~~~~~~~~~~~~~~~~~~~~~ +It's possible for Spack to use certain externally-installed +packages rather than always rebuilding packages. This may be desirable +if machines ship with system packages, such as a customized MPI +that should be used instead of Spack building its own MPI. + +External packages are configured through the ``packages.yaml`` file found +in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` +directory. Here's an example of an external configuration:: + +.. code-block:: yaml + + packages: + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: + path: /opt/openmpi-1.4.3 + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: + path: /opt/openmpi-1.4.3-debug + - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: + path: /opt/openmpi-1.6.5-intel + +This example lists three installations of OpenMPI, one built with gcc, +one built with gcc and debug information, and another built with OpenMPI. +If Spack is asked to build a package that uses one of these MPIs as a +dependency, it link the package to the pre-installed OpenMPI in +the given directory. + +Each ``packages.yaml`` should begin with a ``packages:`` token, followed +by a list of package specs. Specs in the ``packages.yaml`` have at most +one ``path`` tag, which specifies the top-level directory where the +spec is installed. + +Each spec should be as well-defined as reasonably possible. If a +package lacks a spec component, such as missing a compiler or +package version, then Spack will guess the missing component based +on its most-favored packages, and it may guess incorrectly. + +All package versions and compilers listed in ``packages.yaml`` should +have entries in Spack's packages and compiler configuration, even +the package and compiler may not actually be used. + +The packages configuration can tell Spack to use an external location +for certain package versions, but it does not restrict Spack to using +external packages. In the above example, if an OpenMPI 1.8.4 became +available Spack may choose to start building and linking with that version +rather than continue using the pre-installed OpenMPI versions. + +To prevent this, the ``packages.yaml`` configuration also allows packages +to be flagged as non-buildable. The previous example could be modified to +be:: + +.. code-block:: yaml + + packages: + - openmpi: + nobuild: True + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: + path: /opt/openmpi-1.4.3 + - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: + path: /opt/openmpi-1.4.3-debug + - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: + path: /opt/openmpi-1.6.5-intel + +The addition of the ``nobuild`` flag tells Spack that it should never build +its own version of OpenMPI, and it will instead always rely on a pre-built +OpenMPI. Similar to ``path``, ``nobuild`` is specified as a property under +a spec and will prevent building of anything that satisfies that spec. + +The ``nobuild`` does not need to be paired with external packages. +It could also be used alone to forbid versions of packages that may be +buggy or otherwise undesirable. + + Profiling ~~~~~~~~~~~~~~~~~~~~~ From dcddb19e5b55c8c61279a084bdbaa95047236ccb Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 1 Jan 2016 17:35:01 +0100 Subject: [PATCH 013/339] added class decorator to define composite classes --- lib/spack/spack/util/pattern.py | 98 +++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 lib/spack/spack/util/pattern.py diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py new file mode 100644 index 00000000000..8d1584b4c29 --- /dev/null +++ b/lib/spack/spack/util/pattern.py @@ -0,0 +1,98 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import inspect +import collections +import functools + + +def composite(interface=None, method_list=None, container=list): + """ + Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given + interface. + + :param interface: class exposing the interface to which the composite object must conform. Only non-private and + non-special methods will be taken into account + + :param method_list: names of methods that should be part of the composite + + :param container: container for the composite object (default = list). Must fulfill the MutableSequence contract. + The composite class will expose the container API to manage object composition + + :return: class decorator + """ + # Check if container fulfills the MutableSequence contract and raise an exception if it doesn't + # The patched class returned by the decorator will inherit from the container class to expose the + # interface needed to manage objects composition + if not issubclass(container, collections.MutableSequence): + raise TypeError("Container must fulfill the MutableSequence contract") + + # Check if at least one of the 'interface' or the 'method_list' arguments are defined + if interface is None and method_list is None: + raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite") + + def cls_decorator(cls): + # Retrieve the base class of the composite. Inspect its the methods and decide which ones will be overridden + def no_special_no_private(x): + return inspect.ismethod(x) and not x.__name__.startswith('_') + + # Patch the behavior of each of the methods in the previous list. This is done associating an instance of the + # descriptor below to any method that needs to be patched. + class IterateOver(object): + """ + Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the + associated attribute and calls for each of them an attribute with the same name + """ + def __init__(self, name, func=None): + self.name = name + self.func = func + + def __get__(self, instance, owner): + def getter(*args, **kwargs): + for item in instance: + getattr(item, self.name)(*args, **kwargs) + # If we are using this descriptor to wrap a method from an interface, then we must conditionally + # use the `functools.wraps` decorator to set the appropriate fields. + if self.func is not None: + getter = functools.wraps(self.func)(getter) + return getter + + dictionary_for_type_call = {} + # Construct a dictionary with the methods explicitly passed as name + if method_list is not None: + method_list_dict = {name: IterateOver(name) for name in method_list} + dictionary_for_type_call.update(method_list_dict) + # Construct a dictionary with the methods inspected from the interface + if interface is not None: + interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} + interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + dictionary_for_type_call.update(interface_methods_dict) + # Get the methods that are defined in the scope of the composite class and override any previous definition + cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + dictionary_for_type_call.update(cls_method) + # Generate the new class on the fly and return it + wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call) + return wrapper_class + + return cls_decorator From f499f71f64994d1914ca3e8195a79828f442f6e8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 1 Jan 2016 17:36:58 +0100 Subject: [PATCH 014/339] Package : factored out code in do_stage and do_fetch, changed mirror command accordingly --- lib/spack/spack/fetch_strategy.py | 116 ++++++++++++++---------------- lib/spack/spack/mirror.py | 80 ++++++++------------- lib/spack/spack/package.py | 113 +++++++++++++---------------- lib/spack/spack/stage.py | 51 ++++++++++++- 4 files changed, 181 insertions(+), 179 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a9374fb34b9..72b6abe2707 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -55,53 +55,59 @@ from spack.version import Version, ver from spack.util.compression import decompressor_for, extension +import spack.util.pattern as pattern + """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] + def _needs_stage(fun): """Many methods on fetch strategies require a stage to be set using set_stage(). This decorator adds a check for self.stage.""" + @wraps(fun) def wrapper(self, *args, **kwargs): if not self.stage: raise NoStageError(fun) return fun(self, *args, **kwargs) + return wrapper class FetchStrategy(object): """Superclass of all fetch strategies.""" - enabled = False # Non-abstract subclasses should be enabled. + enabled = False # Non-abstract subclasses should be enabled. required_attributes = None # Attributes required in version() args. class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) - def __init__(self): # The stage is initialized late, so that fetch strategies can be constructed # at package construction time. This is where things will be fetched. self.stage = None - def set_stage(self, stage): """This is called by Stage before any of the fetching methods are called on the stage.""" self.stage = stage - # Subclasses need to implement these methods def fetch(self): pass # Return True on success, False on fail. + def check(self): pass # Do checksum. + def expand(self): pass # Expand archive. + def reset(self): pass # Revert to freshly downloaded state. def archive(self, destination): pass # Used to create tarball for mirror. - def __str__(self): # Should be human readable URL. + def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" # This method is used to match fetch strategies to version() @@ -111,6 +117,15 @@ def matches(cls, args): return any(k in args for k in cls.required_attributes) +@pattern.composite(interface=FetchStrategy) +class FetchStrategyComposite(object): + """ + Composite for a FetchStrategy object. Implements the GoF composite pattern. + """ + matches = FetchStrategy.matches + set_stage = FetchStrategy.set_stage + + class URLFetchStrategy(FetchStrategy): """FetchStrategy that pulls source code from a URL for an archive, checks the archive against a checksum,and decompresses the archive. @@ -142,19 +157,19 @@ def fetch(self): tty.msg("Trying to fetch from %s" % self.url) - curl_args = ['-O', # save file to disk - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', self.url,] + curl_args = ['-O', # save file to disk + '-f', # fail on >400 errors + '-D', '-', # print out HTML headers + '-L', self.url, ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: - curl_args.append('-sS') # just errors when not. + curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, return_output=True, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -164,24 +179,23 @@ def fetch(self): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL configuration " + "is bad. If you believe your SSL configuration is bad, you " + "can try running spack -k, which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) - + self.url, "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle @@ -196,7 +210,6 @@ def fetch(self): if not self.archive_file: raise FailedDownloadError(self.url) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -209,7 +222,7 @@ def expand(self): self.stage.chdir() if not self.archive_file: raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) @@ -241,7 +254,6 @@ def expand(self): # Set the wd back to the stage when done. self.stage.chdir() - def archive(self, destination): """Just moves this archive to the destination.""" if not self.archive_file: @@ -252,7 +264,6 @@ def archive(self, destination): shutil.move(self.archive_file, destination) - @_needs_stage def check(self): """Check the downloaded archive against a checksum digest. @@ -263,9 +274,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) - + "%s checksum failed for %s." % (checker.hash_name, self.archive_file), + "Expected %s but got %s." % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -277,12 +287,10 @@ def reset(self): shutil.rmtree(self.stage.source_path, ignore_errors=True) self.expand() - def __repr__(self): url = self.url if self.url else "no url" return "URLFetchStrategy<%s>" % url - def __str__(self): if self.url: return self.url @@ -298,33 +306,30 @@ def __init__(self, name, *rev_types, **kwargs): # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) if not self.url: raise ValueError( - "%s requires %s argument." % (self.__class__, name)) + "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s." % ( + comma_or(rev_types), name)) # Set attributes for each rev type. for rt in rev_types: setattr(self, rt, kwargs.get(rt, None)) - @_needs_stage def check(self): tty.msg("No checksum needed when fetching with %s." % self.name) - @_needs_stage def expand(self): tty.debug("Source fetched with %s is already expanded." % self.name) - @_needs_stage def archive(self, destination, **kwargs): - assert(extension(destination) == 'tar.gz') - assert(self.stage.source_path.startswith(self.stage.path)) + assert (extension(destination) == 'tar.gz') + assert (self.stage.source_path.startswith(self.stage.path)) tar = which('tar', required=True) @@ -338,16 +343,13 @@ def archive(self, destination, **kwargs): self.stage.chdir() tar('-czf', destination, os.path.basename(self.stage.source_path)) - def __str__(self): return "VCS: %s" % self.url - def __repr__(self): return "%s<%s>" % (self.__class__, self.url) - class GitFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a git repository. Use like this in a package: @@ -369,23 +371,20 @@ class GitFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **kwargs) + 'git', 'tag', 'branch', 'commit', **kwargs) self._git = None - @property def git_version(self): vstring = self.git('--version', return_output=True).lstrip('git version ') return Version(vstring) - @property def git(self): if not self._git: self._git = which('git', required=True) return self._git - @_needs_stage def fetch(self): self.stage.chdir() @@ -418,7 +417,7 @@ def fetch(self): if self.branch: args.extend(['--branch', self.branch]) elif self.tag and self.git_version >= ver('1.8.5.2'): - args.extend(['--branch', self.tag]) + args.extend(['--branch', self.tag]) # Try to be efficient if we're using a new enough git. # This checks out only one branch's history @@ -429,7 +428,7 @@ def fetch(self): # Yet more efficiency, only download a 1-commit deep tree if self.git_version >= ver('1.7.1'): try: - self.git(*(args + ['--depth','1', self.url])) + self.git(*(args + ['--depth', '1', self.url])) cloned = True except spack.error.SpackError: # This will fail with the dumb HTTP transport @@ -452,18 +451,15 @@ def fetch(self): self.git('pull', '--tags', ignore_errors=1) self.git('checkout', self.tag) - def archive(self, destination): super(GitFetchStrategy, self).archive(destination, exclude='.git') - @_needs_stage def reset(self): self.stage.chdir_to_source() self.git('checkout', '.') self.git('clean', '-f') - def __str__(self): return "[git] %s" % self.url @@ -484,19 +480,17 @@ class SvnFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **kwargs) + 'svn', 'revision', **kwargs) self._svn = None if self.revision is not None: self.revision = str(self.revision) - @property def svn(self): if not self._svn: self._svn = which('svn', required=True) return self._svn - @_needs_stage def fetch(self): self.stage.chdir() @@ -515,7 +509,6 @@ def fetch(self): self.svn(*args) self.stage.chdir_to_source() - def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" status = self.svn('status', '--no-ignore', return_output=True) @@ -529,23 +522,19 @@ def _remove_untracked_files(self): elif os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) - def archive(self, destination): super(SvnFetchStrategy, self).archive(destination, exclude='.svn') - @_needs_stage def reset(self): self.stage.chdir_to_source() self._remove_untracked_files() self.svn('revert', '.', '-R') - def __str__(self): return "[svn] %s" % self.url - class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -568,10 +557,9 @@ class HgFetchStrategy(VCSFetchStrategy): def __init__(self, **kwargs): super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **kwargs) + 'hg', 'revision', **kwargs) self._hg = None - @property def hg(self): if not self._hg: @@ -597,11 +585,9 @@ def fetch(self): self.hg(*args) - def archive(self, destination): super(HgFetchStrategy, self).archive(destination, exclude='.hg') - @_needs_stage def reset(self): self.stage.chdir() @@ -619,7 +605,6 @@ def reset(self): shutil.move(scrubbed, source_path) self.stage.chdir_to_source() - def __str__(self): return "[hg] %s" % self.url @@ -693,9 +678,10 @@ def __init__(self, msg, long_msg): class FailedDownloadError(FetchError): """Raised wen a download fails.""" + def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -718,12 +704,14 @@ def __init__(self, pkg, version): class ChecksumError(FetchError): """Raised when archive fails to checksum.""" + def __init__(self, message, long_msg=None): super(ChecksumError, self).__init__(message, long_msg) class NoStageError(FetchError): """Raised when fetch operations are called before set_stage().""" + def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef25..44e03a8a450 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -45,12 +45,11 @@ from spack.util.compression import extension, allowed_archive -def mirror_archive_filename(spec): +def mirror_archive_filename(spec, fetcher): """Get the name of the spec's archive in the mirror.""" if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") - fetcher = spec.package.fetcher if isinstance(fetcher, fs.URLFetchStrategy): # If we fetch this version with a URLFetchStrategy, use URL's archive type ext = url.downloaded_file_extension(fetcher.url) @@ -61,9 +60,9 @@ def mirror_archive_filename(spec): return "%s-%s.%s" % (spec.package.name, spec.version, ext) -def mirror_archive_path(spec): +def mirror_archive_path(spec, fetcher): """Get the relative path to the spec's archive within a mirror.""" - return join_path(spec.name, mirror_archive_filename(spec)) + return join_path(spec.name, mirror_archive_filename(spec, fetcher)) def get_matching_versions(specs, **kwargs): @@ -158,68 +157,47 @@ def create(path, specs, **kwargs): everything_already_exists = True for spec in version_specs: pkg = spec.package - - stage = None + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) try: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + for ii, stage in enumerate(pkg.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - if os.path.exists(archive_path): - tty.msg("Already added %s" % spec.format("$_$@")) - else: - everything_already_exists = False - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + everything_already_exists = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - - # Fetch resources if they are associated with the spec - resources = pkg._get_resources() - for resource in resources: - resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) - if os.path.exists(resource_archive_path): - tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) - continue - everything_already_exists = False - resource_stage_folder = pkg._resource_stage(resource) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(resource_stage) - resource.fetcher.fetch() - if not kwargs.get('no_checksum', False): - resource.fetcher.check() - tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) - resource.fetcher.archive(resource_archive_path) - tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) if everything_already_exists: present.append(spec) else: mirrored.append(spec) - except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) else: tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) error.append(spec) - finally: - if stage: - stage.destroy() + pkg.stage.destroy() return (present, mirrored, error) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6673e4f3929..3ccedaa458f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -61,7 +61,7 @@ import spack.util.web import spack.fetch_strategy as fs from spack.version import * -from spack.stage import Stage +from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError @@ -431,23 +431,47 @@ def url_for_version(self, version): return spack.url.substitute_version(self.nearest_url(version), self.url_version(version)) + def _make_resource_stage(self, root_stage, fetcher, resource): + resource_stage_folder = self._resource_stage(resource) + # FIXME : works only for URLFetchStrategy + resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) + stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, + name=resource_stage_folder, mirror_path=resource_mirror) + return stage + + def _make_root_stage(self, fetcher): + # Construct a mirror path (TODO: get this out of package.py) + mp = spack.mirror.mirror_archive_path(self.spec, fetcher) + # Construct a path where the stage should build.. + s = self.spec + stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) + # Build the composite stage + stage = Stage(fetcher, mirror_path=mp, name=stage_name) + return stage + + def _make_stage(self): + # Construct a composite stage on top of the composite FetchStrategy + composite_fetcher = self.fetcher + composite_stage = StageComposite() + resources = self._get_resources() + for ii, fetcher in enumerate(composite_fetcher): + if ii == 0: + # Construct root stage first + stage = self._make_root_stage(fetcher) + else: + # Construct resource stage + resource = resources[ii - 1] # ii == 0 is root! + stage = self._make_resource_stage(composite_stage[0], fetcher, resource) + # Append the item to the composite + composite_stage.append(stage) + return composite_stage @property def stage(self): if not self.spec.concrete: raise ValueError("Can only get a stage for a concrete package.") - if self._stage is None: - # Construct a mirror path (TODO: get this out of package.py) - mp = spack.mirror.mirror_archive_path(self.spec) - - # Construct a path where the stage should build.. - s = self.spec - stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) - - # Build the stage - self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name) - + self._stage = self._make_stage() return self._stage @@ -457,17 +481,25 @@ def stage(self, stage): self._stage = stage + def _make_fetcher(self): + # Construct a composite fetcher that always contains at least one element (the root package). In case there + # are resources associated with the package, append their fetcher to the composite. + root_fetcher = fs.for_package_version(self, self.version) + fetcher = fs.FetchStrategyComposite() # Composite fetcher + fetcher.append(root_fetcher) # Root fetcher is always present + resources = self._get_resources() + for resource in resources: + fetcher.append(resource.fetcher) + return fetcher + @property def fetcher(self): if not self.spec.versions.concrete: - raise ValueError( - "Can only get a fetcher for a package with concrete versions.") - + raise ValueError("Can only get a fetcher for a package with concrete versions.") if not self._fetcher: - self._fetcher = fs.for_package_version(self, self.version) + self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -630,7 +662,7 @@ def remove_prefix(self): def do_fetch(self): - """Creates a stage directory and downloads the taball for this package. + """Creates a stage directory and downloads the tarball for this package. Working directory will be set to the stage directory. """ if not self.spec.concrete: @@ -656,20 +688,6 @@ def do_fetch(self): self.stage.fetch() - ########## - # Fetch resources - resources = self._get_resources() - for resource in resources: - resource_stage_folder = self._resource_stage(resource) - # FIXME : works only for URLFetchStrategy - resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) - resource.fetcher.set_stage(resource_stage) - # Delegate to stage object to trigger mirror logic - resource_stage.fetch() - resource_stage.check() - ########## - self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: @@ -681,39 +699,10 @@ def do_stage(self): if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - def _expand_archive(stage, name=self.name): - archive_dir = stage.source_path - if not archive_dir: - stage.expand_archive() - tty.msg("Created stage in %s." % stage.path) - else: - tty.msg("Already staged %s in %s." % (name, stage.path)) - - self.do_fetch() - _expand_archive(self.stage) - - ########## - # Stage resources in appropriate path - resources = self._get_resources() - for resource in resources: - stage = resource.fetcher.stage - _expand_archive(stage, resource.name) - # Turn placement into a dict with relative paths - placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement - if not isinstance(placement, dict): - placement = {'': placement} - # Make the paths in the dictionary absolute and link - for key, value in placement.iteritems(): - link_path = join_path(self.stage.source_path, resource.destination, value) - source_path = join_path(stage.source_path, key) - if not os.path.exists(link_path): - # Create a symlink - os.symlink(source_path, link_path) - ########## + self.stage.expand_archive() self.stage.chdir_to_source() - def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 754344fc016..9b4c8fdc1d4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -30,6 +30,8 @@ import llnl.util.tty as tty from llnl.util.filesystem import * +import spack.util.pattern as pattern + import spack import spack.config import spack.fetch_strategy as fs @@ -40,7 +42,7 @@ class Stage(object): - """A Stage object manaages a directory where some source code is + """A Stage object manages a directory where some source code is downloaded and built before being installed. It handles fetching the source code, either as an archive to be expanded or by checking it out of a repository. A stage's lifecycle @@ -276,7 +278,12 @@ def expand_archive(self): archive. Fail if the stage is not set up or if the archive is not yet downloaded. """ - self.fetcher.expand() + archive_dir = self.source_path + if not archive_dir: + self.fetcher.expand() + tty.msg("Created stage in %s." % self.path) + else: + tty.msg("Already staged %s in %s." % (self.name, self.path)) def chdir_to_source(self): @@ -310,6 +317,46 @@ def destroy(self): os.chdir(os.path.dirname(self.path)) +class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): + super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) + self.root_stage = root + self.resource = resource + + def expand_archive(self): + super(ResourceStage, self).expand_archive() + root_stage = self.root_stage + resource = self.resource + placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + if not isinstance(placement, dict): + placement = {'': placement} + # Make the paths in the dictionary absolute and link + for key, value in placement.iteritems(): + link_path = join_path(root_stage.source_path, resource.destination, value) + source_path = join_path(self.source_path, key) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(source_path, link_path) + + +@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) +class StageComposite: + """ + Composite for Stage type objects. The first item in this composite is considered to be the root package, and + operations that return a value are forwarded to it. + """ + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + + def chdir_to_source(self): + return self[0].chdir_to_source() + + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" def __init__(self, path): From d95d169ac50a3c253ef9712782626af1e1610ca7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 2 Jan 2016 19:04:06 +0100 Subject: [PATCH 015/339] fixed broken unit tests --- lib/spack/spack/test/install.py | 7 +++++-- lib/spack/spack/util/pattern.py | 26 ++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 1ef4171fb2f..bb7d3f4fd4c 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -31,7 +31,7 @@ import spack from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.directory_layout import YamlDirectoryLayout from spack.util.executable import which from spack.test.mock_packages_test import * @@ -81,7 +81,10 @@ def test_install_and_uninstall(self): pkg = spack.db.get(spec) # Fake the URL for the package so it downloads from a file. - pkg.fetcher = URLFetchStrategy(self.repo.url) + + fetcher = FetchStrategyComposite() + fetcher.append(URLFetchStrategy(self.repo.url)) + pkg.fetcher = fetcher try: pkg.do_install() diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 8d1584b4c29..73c1e26aa5d 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -81,17 +81,35 @@ def getter(*args, **kwargs): dictionary_for_type_call = {} # Construct a dictionary with the methods explicitly passed as name if method_list is not None: - method_list_dict = {name: IterateOver(name) for name in method_list} + # python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list} + method_list_dict = {} + for name in method_list: + method_list_dict[name] = IterateOver(name) dictionary_for_type_call.update(method_list_dict) # Construct a dictionary with the methods inspected from the interface if interface is not None: - interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} - interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + ########## + # python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} + interface_methods = {} + for name, method in inspect.getmembers(interface, predicate=no_special_no_private): + interface_methods[name] = method + ########## + # python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + interface_methods_dict = {} + for name, method in interface_methods.iteritems(): + interface_methods_dict[name] = IterateOver(name, method) + ########## dictionary_for_type_call.update(interface_methods_dict) # Get the methods that are defined in the scope of the composite class and override any previous definition - cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + ########## + # python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + cls_method = {} + for name, method in inspect.getmembers(cls, predicate=inspect.ismethod): + cls_method[name] = method + ########## dictionary_for_type_call.update(cls_method) # Generate the new class on the fly and return it + # FIXME : inherit from interface if we start to use ABC classes? wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call) return wrapper_class From bf4231dd4b906f20d522b9df5f2cae2ba6291174 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 2 Jan 2016 22:48:14 +0100 Subject: [PATCH 016/339] depends_on and when multimethod support boolean --- lib/spack/spack/directives.py | 6 +++- lib/spack/spack/multimethod.py | 9 ++++-- lib/spack/spack/util/architecture.py | 46 +++++++++++++++++++++++++++ var/spack/packages/openssl/package.py | 15 +++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index aa9fbd8d339..02477bb904a 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs): def _depends_on(pkg, spec, when=None): - if when is None: + # If when is False do nothing + if when is False: + return + # If when is None or True make sure the condition is always satisfied + if when is None or when is True: when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df9b9b2ab14..51c6a8e89d9 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -193,10 +193,11 @@ def install(self, prefix): platform-specific versions. There's not much we can do to get around this because of the way decorators work. """ -class when(object): def __init__(self, spec): pkg = get_calling_module_name() - self.spec = parse_anonymous_spec(spec, pkg) + if spec is True: + spec = pkg + self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None def __call__(self, method): # Get the first definition of the method in the calling scope @@ -207,7 +208,9 @@ def __call__(self, method): if not type(original_method) == SpecMultiMethod: original_method = SpecMultiMethod(original_method) - original_method.register(self.spec, method) + if self.spec is not None: + original_method.register(self.spec, method) + return original_method diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py new file mode 100644 index 00000000000..a020c74a7ac --- /dev/null +++ b/lib/spack/spack/util/architecture.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import sys + + +def os_is_in(*args): + """ + Return True if any element in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return any(map(lambda x: x == sys.platform, args)) + + +def os_is_not_in(*args): + """ + Return True if none of the elements in the list is equal to sys.platform, False otherwise + + :param args: list of names to be checked + :return: True or False + """ + return not os_is_in(*args) diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index 40648fca492..1fa288a36dc 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -1,5 +1,7 @@ from spack import * +import spack.util.architecture as arch + class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -26,3 +28,16 @@ def install(self, spec, prefix): make() make("install") + + @when(arch.os_is_in('darwin')) + def install(self, spec, prefix): + perl = which('perl') + perl("./Configure", + "--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared", + "darwin64-x86_64-cc") + filter_file(r'-arch x86_64', '', 'Makefile') + From 50727527bc69a1b8ed26aa84a40039086db33827 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 11 Jan 2016 15:51:02 -0800 Subject: [PATCH 017/339] This commit makes the following changes: There are two sensible defaults for building boost libraries: build all of them or build none of them. Previously the Spack boost package took the first approach. This commit changes to building no libraries by default. The user can specify which libraries they need using variants (e.g. +iostreams to compile the boost iostreams library). If no libraries are built then a header-only install is performed (no compilation, just copy header files to prefix). The consequence of this change is that packages which specify a dependency on boost may now fail (until they are updated to specify exactly which boost libraries they need compiled). The user may now specify whether to build shared libraries (static libraries are always built) and whether to build libraries with/out multi-threading support (default is to only build with multi-threading support). The executable on the user-config.jam toolset line is set to Spack's cc script. Before, without this, the desired toolset was used but Spack deferred to the boost build system to choose the compiler version. bzip2 and zlib are always specified as dependencies when iostreams is built (before this could be controlled with the +compression variant). --- var/spack/packages/boost/package.py | 90 ++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 3427b74ad68..5fb6c9dc049 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -1,4 +1,5 @@ from spack import * +import spack class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -44,15 +45,35 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - variant('debug', default=False, description='Switch to the debug version of Boost') - variant('python', default=False, description='Activate the component Boost.Python') - variant('mpi', default=False, description='Activate the component Boost.MPI') - variant('compression', default=True, description='Activate the compression Boost.iostreams') + libs = ['chrono', + 'date_time', + 'filesystem', + 'iostreams', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'thread', + 'wave', + 'mpi', + 'python'] + for lib in libs: + variant(lib, default=False, description="Compile with {0} library" + .format(lib)) + + variant('debug', default=False, description='Switch to the debug version of Boost') + variant('shared', default=True, description="Additionally build shared libraries") + variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") + variant('singlethreaded', default=False, description="Build single-threaded versions of libraries") + variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") + + depends_on('icu', when='+regex_icu') depends_on('python', when='+python') depends_on('mpi', when='+mpi') - depends_on('bzip2', when='+compression') - depends_on('zlib', when='+compression') + depends_on('bzip2', when='+iostreams') + depends_on('zlib', when='+iostreams') def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" @@ -77,22 +98,20 @@ def determine_toolset(self, spec): # fallback to gcc if no toolset found return 'gcc' - def determine_bootstrap_options(self, spec, options): - options.append('--with-toolset=%s' % self.determine_toolset(spec)) + def determine_bootstrap_options(self, spec, withLibs, options): + boostToolsetId = self.determine_toolset(spec) + options.append('--with-toolset=%s' % boostToolsetId) + options.append("--with-libraries=%s" % ','.join(withLibs)) - without_libs = [] - if '~mpi' in spec: - without_libs.append('mpi') - if '~python' in spec: - without_libs.append('python') - else: + if '+python' in spec: options.append('--with-python=%s' % join_path(spec['python'].prefix.bin, 'python')) - if without_libs: - options.append('--without-libraries=%s' % ','.join(without_libs)) - with open('user-config.jam', 'w') as f: + compiler_wrapper = join_path(spack.build_env_path, 'c++') + f.write("using {0} : : {1} ;\n".format(boostToolsetId, + compiler_wrapper)) + if '+mpi' in spec: f.write('using mpi : %s ;\n' % join_path(spec['mpi'].prefix.bin, 'mpicxx')) @@ -107,12 +126,7 @@ def determine_b2_options(self, spec, options): else: options.append('variant=release') - if '~compression' in spec: - options.extend([ - '-s', 'NO_BZIP2=1', - '-s', 'NO_ZLIB=1']) - - if '+compression' in spec: + if '+iostreams' in spec: options.extend([ '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, @@ -120,20 +134,44 @@ def determine_b2_options(self, spec, options): '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, ]) + linkTypes = ['static'] + if '+shared' in spec: + linkTypes.append('shared') + + #TODO: at least one of these two options must be active + threadingOpts = [] + if '+multithreaded' in spec: + threadingOpts.append('multi') + if '+singlethreaded' in spec: + threadingOpts.append('single') + options.extend([ 'toolset=%s' % self.determine_toolset(spec), - 'link=static,shared', - 'threading=single,multi', + 'link=%s' % ','.join(linkTypes), + 'threading=%s' % ','.join(threadingOpts), '--layout=tagged']) def install(self, spec, prefix): + withLibs = list() + for lib in Boost.libs: + if "+{0}".format(lib) in spec: + withLibs.append(lib) + if not withLibs: + # if no libraries are specified for compilation, then you dont have + # to configure/build anything, just copy over to the prefix directory. + src = join_path(self.stage.source_path, 'boost') + mkdirp(join_path(prefix, 'include')) + dst = join_path(prefix, 'include', 'boost') + install_tree(src, dst) + return + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' bootstrap = Executable('./bootstrap.sh') bootstrap_options = ['--prefix=%s' % prefix] - self.determine_bootstrap_options(spec, bootstrap_options) + self.determine_bootstrap_options(spec, withLibs, bootstrap_options) bootstrap(*bootstrap_options) From 9b3e0255f4e8c1ec02aae2a75a11383ad274507d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 11:55:42 -0600 Subject: [PATCH 018/339] Add HDF package --- var/spack/packages/hdf/package.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 var/spack/packages/hdf/package.py diff --git a/var/spack/packages/hdf/package.py b/var/spack/packages/hdf/package.py new file mode 100644 index 00000000000..7882665dbaf --- /dev/null +++ b/var/spack/packages/hdf/package.py @@ -0,0 +1,35 @@ +from spack import * + +class Hdf(Package): + """HDF4 (also known as HDF) is a library and multi-object + file format for storing and managing data between machines.""" + + homepage = "https://www.hdfgroup.org/products/hdf4/" + url = "https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.11/src/hdf-4.2.11.tar.gz" + list_url = "https://www.hdfgroup.org/ftp/HDF/releases/" + list_depth = 3 + + version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') + + depends_on("jpeg") + depends_on("szip@2.1") + depends_on("zlib") + + + def url_for_version(self, version): + return "https://www.hdfgroup.org/ftp/HDF/releases/HDF" + str(version) + "/src/hdf-" + str(version) + ".tar.gz" + + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-szlib=%s' % spec['szip'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', + '--enable-fortran', + '--disable-shared', + '--enable-static', + '--enable-production') + + make() + make("install") From 5520ce3c4a46476f5eec7a340471dd044932ac06 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 12:19:03 -0600 Subject: [PATCH 019/339] Removed unnecessary URLs --- var/spack/packages/xz/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py index ba6c9733a7e..fdcac953458 100644 --- a/var/spack/packages/xz/package.py +++ b/var/spack/packages/xz/package.py @@ -8,11 +8,9 @@ class Xz(Package): homepage = "http://tukaani.org/xz/" url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" - version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', - url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') - version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', - url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') - + version('5.2.0', '867cc8611760240ebf3440bd6e170bb9') + version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From 6985d2c914159f7978c304530f7fb1eb38b2b78d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:00:05 -0600 Subject: [PATCH 020/339] Add url_for_version function to calculate url --- var/spack/packages/hwloc/package.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 7ebede76a3b..60b315119ba 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -14,15 +14,18 @@ class Hwloc(Package): efficiently.""" homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + list_url = "http://www.open-mpi.org/software/hwloc/" + list_depth = 3 - version('1.11.2', '486169cbe111cdea57be12638828ebbf', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.2.tar.bz2') - version('1.11.1', '002742efd3a8431f98d6315365a2b543', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') - version('1.9', '1f9f9155682fe8946a97c08896109508') + version('1.11.2', '486169cbe111cdea57be12638828ebbf') + version('1.11.1', '002742efd3a8431f98d6315365a2b543') + version('1.9', '1f9f9155682fe8946a97c08896109508') depends_on('libpciaccess') + def url_for_version(self, version): + return "http://www.open-mpi.org/software/hwloc/v%s/downloads/hwloc-%s.tar.gz" % (version.up_to(2), version) + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 180c673c782d10864dce6cd7df43465dd05579a1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 12 Jan 2016 13:21:53 -0600 Subject: [PATCH 021/339] Add different URL for older source code --- var/spack/packages/mpc/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 50477a0ccba..65c0ba7ea13 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -34,8 +34,14 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp") - depends_on("mpfr") + depends_on("gmp@4.3.2:") + depends_on("mpfr@2.4.2:") + + def url_for_version(self, version): + if version < Version("1.0.1"): + return "http://www.multiprecision.org/mpc/download/mpc-%s.tar.gz" % version + else: + return "ftp://ftp.gnu.org/gnu/mpc/mpc-%s.tar.gz" % version def install(self, spec, prefix): configure("--prefix=%s" % prefix) From f9fbb57d3182bf0e9c39e4b7c0a958c6ba87a2fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 13 Jan 2016 12:32:04 -0600 Subject: [PATCH 022/339] Add NAG Fortran Compiler support --- lib/spack/env/cc | 4 ++-- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/compilers/nag.py | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 lib/spack/spack/compilers/nag.py diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 0966277a91e..aacba996b39 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -94,11 +94,11 @@ case "$command" in command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf) + f77|gfortran|ifort|pgf77|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 66e608cf79f..a6198fc1ce2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -45,7 +45,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] def _auto_compiler_spec(function): def converter(cspec_like): diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py new file mode 100644 index 00000000000..2dfc97af739 --- /dev/null +++ b/lib/spack/spack/compilers/nag.py @@ -0,0 +1,24 @@ +class Nag(Compiler): + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['nagfor'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['nagfor'] + + @classmethod + def default_version(self, comp): + """The '-V' option works for nag compilers. + Output looks like this:: + + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux + Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K. + """ + return get_compiler_version( + comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') From 7d0256c9fb5c3b8824cdeb0455d2a3e3b3f1eeb4 Mon Sep 17 00:00:00 2001 From: Nicola Varini Date: Fri, 15 Jan 2016 10:27:38 +0100 Subject: [PATCH 023/339] Quantum-ESPRESSO package --- var/spack/packages/espresso/package.py | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 var/spack/packages/espresso/package.py diff --git a/var/spack/packages/espresso/package.py b/var/spack/packages/espresso/package.py new file mode 100644 index 00000000000..f9df74c0c08 --- /dev/null +++ b/var/spack/packages/espresso/package.py @@ -0,0 +1,61 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install espresso +# +# You can always get back here to change things with: +# +# spack edit espresso +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Espresso(Package): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://quantum-espresso.org" + url = "http://www.qe-forge.org/gf/download/frsrelease/199/855/espresso-5.2.1.tar.gz" + + version('5.2.1', 'da3ec5302e4343804e65de60f6004c2d') + variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + depends_on('mpi', when='+mpi') + + +# def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. +# configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here +# make() +# make("install") + + def install(self, spec, prefix): + # TAU isn't happy with directories that have '@' in the path. Sigh. + + # TAU configure, despite the name , seems to be a manually written script (nothing related to autotools). + # As such it has a few #peculiarities# that make this build quite hackish. + options = ["-prefix=%s" % prefix, + "--enable-parallel"] + + if '+openmp' in spec: + options.append('--enable-openmp') + + if '+scalapack' in spec: + options.append('--with-scalapack=yes') + + configure(*options) + make("all") + make("install") + From 7fc308ed26d7abfcb71af8165523b421a1f315c0 Mon Sep 17 00:00:00 2001 From: Nicola Varini Date: Fri, 15 Jan 2016 17:04:25 +0100 Subject: [PATCH 024/339] Update to version 5.3 --- var/spack/packages/espresso/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/espresso/package.py b/var/spack/packages/espresso/package.py index f9df74c0c08..ce5dcc2accc 100644 --- a/var/spack/packages/espresso/package.py +++ b/var/spack/packages/espresso/package.py @@ -20,9 +20,9 @@ class Espresso(Package): """FIXME: put a proper description of your package here.""" # FIXME: add a proper url for your package's homepage here. homepage = "http://quantum-espresso.org" - url = "http://www.qe-forge.org/gf/download/frsrelease/199/855/espresso-5.2.1.tar.gz" + url = "http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz" - version('5.2.1', 'da3ec5302e4343804e65de60f6004c2d') + version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') From d22cf1aed1d033610481cb451909c93916848ccc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 15 Jan 2016 18:07:41 -0800 Subject: [PATCH 025/339] 1. raise an exception if the multithreaded and singlethreaded options are both disabled 2. invoke the b2 installation once for each enabled threading option (apparently the install fails if a single call has both options enabled for mpi) --- var/spack/packages/boost/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 5fb6c9dc049..1992d4d39a5 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -138,18 +138,20 @@ def determine_b2_options(self, spec, options): if '+shared' in spec: linkTypes.append('shared') - #TODO: at least one of these two options must be active threadingOpts = [] if '+multithreaded' in spec: threadingOpts.append('multi') if '+singlethreaded' in spec: threadingOpts.append('single') + if not threadingOpts: + raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") options.extend([ 'toolset=%s' % self.determine_toolset(spec), 'link=%s' % ','.join(linkTypes), - 'threading=%s' % ','.join(threadingOpts), '--layout=tagged']) + + return threadingOpts def install(self, spec, prefix): withLibs = list() @@ -181,6 +183,10 @@ def install(self, spec, prefix): b2 = Executable(b2name) b2_options = ['-j', '%s' % make_jobs] - self.determine_b2_options(spec, b2_options) + threadingOpts = self.determine_b2_options(spec, b2_options) - b2('install', *b2_options) + # In theory it could be done on one call but it fails on + # Boost.MPI if the threading options are not separated. + for threadingOpt in threadingOpts: + b2('install', 'threading=%s' % threadingOpt, *b2_options) + From 58162ec17070db5476b3aeffe1ef40d676a62949 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 18 Jan 2016 16:03:41 -0600 Subject: [PATCH 026/339] Add verbs variant, remove deprecated configure flags --- var/spack/packages/mpich/package.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index 00b7dfda751..128a1f44aef 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -41,6 +41,8 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') @@ -56,14 +58,20 @@ def install(self, spec, prefix): config_args = ["--prefix=" + prefix, "--enable-shared"] + # Variants + if '+verbs' in spec: + config_args.append("--with-ibverbs") + else: + config_args.append("--without-ibverbs") + # TODO: Spack should make it so that you can't actually find # these compilers if they're "disabled" for the current # compiler configuration. if not self.compiler.f77: - config_args.append("--disable-f77") + config_args.append("--disable-fortran=f77") if not self.compiler.fc: - config_args.append("--disable-fc") + config_args.append("--disable-fortran=fc") configure(*config_args) make() From f48fc0d8b184e26accc570910b8f4244d4d9e63f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 09:43:28 -0600 Subject: [PATCH 027/339] Revert deprecated configure flags --- var/spack/repos/builtin/packages/mpich/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 128a1f44aef..c856cfe277f 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -68,10 +68,10 @@ def install(self, spec, prefix): # these compilers if they're "disabled" for the current # compiler configuration. if not self.compiler.f77: - config_args.append("--disable-fortran=f77") + config_args.append("--disable-f77") if not self.compiler.fc: - config_args.append("--disable-fortran=fc") + config_args.append("--disable-fc") configure(*config_args) make() From 0d42cdaffdf12fa3717135b07fade947dcb6b3b7 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 10:09:25 -0600 Subject: [PATCH 028/339] Merge branch 'develop' into features/hdf --- var/spack/{ => repos/builtin}/packages/hdf/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/hdf/package.py (100%) diff --git a/var/spack/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py similarity index 100% rename from var/spack/packages/hdf/package.py rename to var/spack/repos/builtin/packages/hdf/package.py From e121faffa1974bff9570eb8b7a2ba83fdd624f0e Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 19 Jan 2016 17:26:26 +0100 Subject: [PATCH 029/339] eigen : added package --- .../repos/builtin/packages/eigen/package.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 var/spack/repos/builtin/packages/eigen/package.py diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py new file mode 100644 index 00000000000..44ee6819f58 --- /dev/null +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Eigen(Package): + """ + Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms + """ + + homepage = 'http://eigen.tuxfamily.org/' + url = 'http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2' + + version('3.2.7', 'cc1bacbad97558b97da6b77c9644f184', url='http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2') + + variant('debug', default=False, description='Builds the library in debug mode') + + variant('metis', default=True, description='Enables metis backend') + variant('scotch', default=True, description='Enables scotch backend') + variant('fftw', default=True, description='Enables FFTW backend') + + # TODO : dependency on SuiteSparse, googlehash, superlu, adolc missing + + depends_on('metis', when='+metis') + depends_on('scotch', when='+scotch') + depends_on('fftw', when='+fftw') + + depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 + depends_on('gmp') + + def install(self, spec, prefix): + + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") From 98706d31dca0773c51acfabb8f1a399d4720a875 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 12:46:15 -0600 Subject: [PATCH 030/339] Various updates to MVAPICH2 package Use url_for_version function to specify urls Add version 2.2b Add dedicated MRAIL variant Fix typo Make fortran compiler flags more specific --- .../builtin/packages/mvapich2/package.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 23a11b3171a..af5ed1b0888 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -4,15 +4,13 @@ class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" + url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2b.tar.gz" - version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2a.tar.gz') + version('2.2b', '5651e8b7a72d7c77ca68da48f3a5d108') + version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2') + version('2.0', '9fbb68a4111a8b6338e476dc657388b4') + version('1.9', '5dc58ed08fd3142c260b70fe297e127c') - version('2.0', '9fbb68a4111a8b6338e476dc657388b4', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 @@ -41,16 +39,25 @@ class Mvapich2(Package): NEMESISIBTCP = 'nemesisibtcp' NEMESISIB = 'nemesisib' NEMESIS = 'nemesis' + MRAIL = 'mrail' SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP) variant(PSM, default=False, description='Configures a build for QLogic PSM-CH3') variant(SOCK, default=False, description='Configures a build for TCP/IP-CH3') variant(NEMESISIBTCP, default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') variant(NEMESISIB, default=False, description='Configures a build for OFA-IB-Nemesis') variant(NEMESIS, default=False, description='Configures a build for TCP/IP-Nemesis') + variant(MRAIL, default=False, description='Configures a build for OFA-IB-CH3') ########## # FIXME : CUDA support is missing + def url_for_version(self, version): + base_url = "http://mvapich.cse.ohio-state.edu/download" + if version < Version('2.0'): + return "%s/mvapich2/mv2/mvapich2-%s.tar.gz" % (base_url, version) + else: + return "%s/mvapich/mv2/mvapich2-%s.tar.gz" % (base_url, version) + @staticmethod def enabled(x): """ @@ -117,7 +124,7 @@ def set_network_type(self, spec, configure_args): if count > 1: raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') - # From here on I can suppose that ony one variant has been selected + # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] elif self.enabled(Mvapich2.SOCK) in spec: @@ -128,7 +135,7 @@ def set_network_type(self, spec, configure_args): network_options = ["--with-device=ch3:nemesis:ib"] elif self.enabled(Mvapich2.NEMESIS) in spec: network_options = ["--with-device=ch3:nemesis"] - else: + elif self.enabled(Mvapich2.MRAIL) in spec: network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] configure_args.extend(network_options) @@ -141,7 +148,14 @@ def install(self, spec, prefix): "--enable-romio", "--disable-silent-rules", ] - if not self.compiler.f77 and not self.compiler.fc: + + if self.compiler.f77 and self.compiler.fc: + configure_args.append("--enable-fortran=all") + elif self.compiler.f77: + configure_args.append("--enable-fortran=f77") + elif self.compiler.fc: + configure_args.append("--enable-fortran=fc") + else: configure_args.append("--enable-fortran=none") # Set the type of the build (debug, release) From e6aa2610bc4314cab09de61b4f7b632409299fb2 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:07:07 -0600 Subject: [PATCH 031/339] Add symlink for NAG Fortran Compiler --- lib/spack/env/nag/nagfor | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/spack/env/nag/nagfor diff --git a/lib/spack/env/nag/nagfor b/lib/spack/env/nag/nagfor new file mode 120000 index 00000000000..82c2b8e90a3 --- /dev/null +++ b/lib/spack/env/nag/nagfor @@ -0,0 +1 @@ +../cc \ No newline at end of file From 83de658ee42e08a27ce7d85d295955f4809c03dc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:28:35 -0600 Subject: [PATCH 032/339] Modify nag.py to match new compiler package structure --- lib/spack/spack/compilers/nag.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 2dfc97af739..f1cc6be0d5c 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,3 +1,5 @@ +from spack.compiler import * + class Nag(Compiler): # Subclasses use possible names of C compiler cc_names = [] @@ -11,6 +13,13 @@ class Nag(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['nagfor'] + # Named wrapper links within spack.build_env_path + link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml + 'cc' : 'cc', + 'cxx' : 'cxx', + 'f77' : 'nag/nagfor', + 'fc' : 'nag/nagfor' } + @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. From 6c9800bcc395ecbc319f4a3f6242efee4b972470 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 19 Jan 2016 15:22:16 -0500 Subject: [PATCH 033/339] typos: fix some letter transposes --- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index a0c78bf755b..d6b23d6a08e 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -37,7 +37,7 @@ def setup_parser(subparser): help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( '-a', '--all', action='store_true', - help="Deactivate all extensions of an extendable pacakge, or " + help="Deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 0fa18db34b8..9cbe7de44a3 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -489,7 +489,7 @@ def remove(self, spec): 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts - and remvoes them if they are no longer needed. + and removes them if they are no longer needed. """ # Take a lock around the entire removal. From ab27fdc692193fa2deeacfbb2aabc1f6feb06f6d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 09:35:15 +0100 Subject: [PATCH 034/339] elpa : commit partial package. Blocked by missing scalapack --- .../repos/builtin/packages/elpa/package.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 var/spack/repos/builtin/packages/elpa/package.py diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py new file mode 100644 index 00000000000..6825056a04b --- /dev/null +++ b/var/spack/repos/builtin/packages/elpa/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Elpa(Package): + """ + Eigenvalue solvers for Petaflop-Applications (ELPA) + """ + + homepage = 'http://elpa.mpcdf.mpg.de/' + url = 'http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz' + + version('2015.11.001', 'de0f35b7ee7c971fd0dca35c900b87e6', url='http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 72d5cdf9abaee156d80dd027911a6cadbfebd7bb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 11:19:36 +0100 Subject: [PATCH 035/339] fixed two minor typos --- lib/spack/docs/site_configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index a7211a9d95b..8ab4e89dfce 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -76,9 +76,9 @@ directory. Here's an example of an external configuration:: path: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, -one built with gcc and debug information, and another built with OpenMPI. +one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a -dependency, it link the package to the pre-installed OpenMPI in +dependency, it links the package to the pre-installed OpenMPI in the given directory. Each ``packages.yaml`` should begin with a ``packages:`` token, followed From 65f65ec4e47289e15e5d6a25913beb8e6bb4831b Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 10:03:24 +0100 Subject: [PATCH 036/339] Adding MUMPS and ScaLAPACK --- var/spack/packages/mumps/Makefile.inc | 41 ++++++ var/spack/packages/mumps/package.py | 121 ++++++++++++++++++ .../packages/netlib-scalapack/package.py | 43 +++++++ 3 files changed, 205 insertions(+) create mode 100644 var/spack/packages/mumps/Makefile.inc create mode 100644 var/spack/packages/mumps/package.py create mode 100644 var/spack/packages/netlib-scalapack/package.py diff --git a/var/spack/packages/mumps/Makefile.inc b/var/spack/packages/mumps/Makefile.inc new file mode 100644 index 00000000000..4b6696db5c3 --- /dev/null +++ b/var/spack/packages/mumps/Makefile.inc @@ -0,0 +1,41 @@ +LPORDDIR = $(topdir)/PORD/lib/ +IPORD = -I$(topdir)/PORD/include/ +LPORD = -L$(LPORDDIR) -lpord + +ORDERINGSC = $(ORDERINGSF) +LORDERINGS = $(LMETIS) $(LPORD) $(LSCOTCH) +IORDERINGSF = $(ISCOTCH) +IORDERINGSC = $(IMETIS) $(IPORD) $(ISCOTCH) + +PLAT = +LIBEXT = .a +OUTC = -o +OUTF = -o +RM = /bin/rm -f +AR = ar vr +RANLIB = ranlib + +INCSEQ = -I$(topdir)/libseq +LIBSEQ = -L$(topdir)/libseq -lmpiseq + +INCPAR = +LIBPAR = $(SCALAP) + +LIBOTHERS = -lpthread + +#Preprocessor defs for calling Fortran from C (-DAdd_ or -DAdd__ or -DUPPER) +CDEFS = -DAdd_ + +#Sequential: +ifeq ($(MUMPS_TYPE),seq) +INCS = $(INCSEQ) +LIBS = $(LIBSEQ) +LIBSEQNEEDED = libseqneeded +endif + +#Parallel: +ifeq ($(MUMPS_TYPE),par) +INCS = $(INCPAR) +LIBS = $(LIBPAR) +LIBSEQNEEDED = +endif diff --git a/var/spack/packages/mumps/package.py b/var/spack/packages/mumps/package.py new file mode 100644 index 00000000000..b4e38c02de4 --- /dev/null +++ b/var/spack/packages/mumps/package.py @@ -0,0 +1,121 @@ +from spack import * +import os + + +class Mumps(Package): + """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" + + homepage = "http://mumps.enseeiht.fr" + url = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz" + + version('5.0.1', 'b477573fdcc87babe861f62316833db0') + + variant('mpi', default=True, description='Activate the compilation of MUMPS with the MPI support') + variant('scotch', default=False, description='Activate Scotch as a possible ordering library') + variant('ptscotch', default=False, description='Activate PT-Scotch as a possible ordering library') + variant('metis', default=False, description='Activate Metis as a possible ordering library') + variant('parmetis', default=False, description='Activate Parmetis as a possible ordering library') + variant('double', default=True, description='Activate dmumps') + variant('float', default=True, description='Activate smumps') + variant('complex', default=True, description='Activate cmumps and/or zmumps') + variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') + variant('double', default=False, description='Use double precision floating point types') + + + depends_on('scotch + esmumps', when='~ptscotch+scotch') + depends_on('scotch + esmumps + mpi', when='+ptscotch') + depends_on('metis', when='~parmetis+metis') + depends_on('parmetis', when="+parmetis") + depends_on('blas') + depends_on('scalapack', when='+mpi') + depends_on('mpi', when='+mpi') + + def patch(self): + if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + + makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + + orderings = ['-Dpord'] + + if '+ptscotch' in self.spec or '+scotch' in self.spec: + join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') + makefile_conf.extend( + ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + orderings.append('-Dscotch') + if '+ptscotch' in self.spec: + orderings.append('-Dptscotch') + + if '+parmetis' in self.spec or '+metis' in self.spec: + libname = 'parmetis' if '+parmetis' in self.spec else 'metis' + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec[libname].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec[libname].prefix.lib, libname)]) + + orderings.append('-Dmetis') + if '+parmetis' in self.spec: + orderings.append('-Dparmetis') + + makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + + if '+idx64' in self.spec: + makefile_conf.extend( + ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', + 'OPTL = -O ', + 'OPTC = -O -DINTSIZE64']) + else: + makefile_conf.extend( + ['OPTF = -O -DALLOW_NON_INIT', + 'OPTL = -O ', + 'OPTC = -O ']) + + + if '+mpi' in self.spec: + makefile_conf.extend( + ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), + "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "SCALAP = %s" % self.spec['scalapack'].fc_link]) + else: + makefile_conf.extend( + ["CC = cc", + "FC = fc", + "FL = fc", + "MUMPS_TYPE = seq"]) + + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), + 'Makefile.inc') + with open(makefile_inc_template, "r") as fh: + makefile_conf.extend(fh.read().split('\n')) + + with working_dir('.'): + with open("Makefile.inc", "w") as fh: + makefile_inc = '\n'.join(makefile_conf) + fh.write(makefile_inc) + + + + def install(self, spec, prefix): + make_libs = [] + + # the coice to compile ?examples is to have kind of a sanity + # check on the libraries generated. + if '+float' in spec: + make_libs.append('sexamples') + if '+complex' in spec: + make_libs.append('cexamples') + + if '+double' in spec: + make_libs.append('dexamples') + if '+complex' in spec: + make_libs.append('zexamples') + + make(*make_libs) + + install_tree('lib', prefix.lib) + install_tree('include', prefix.include) + if '~mpi' in spec: + install('libseq/libmpiseq.a', prefix.lib) diff --git a/var/spack/packages/netlib-scalapack/package.py b/var/spack/packages/netlib-scalapack/package.py new file mode 100644 index 00000000000..69df4d9da75 --- /dev/null +++ b/var/spack/packages/netlib-scalapack/package.py @@ -0,0 +1,43 @@ +from spack import * + +class NetlibScalapack(Package): + """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" + + homepage = "http://www.netlib.org/scalapack/" + url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" + + version('2.0.2', '2f75e600a2ba155ed9ce974a1c4b536f') + version('2.0.1', '17b8cde589ea0423afe1ec43e7499161') + version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') + + variant('shared', default=True, description='Build the shared library version') + + provides('scalapack') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + + def install(self, spec, prefix): + options = [ + "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', + "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_Fortran_FLAGS=-fPIC", + ] + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *options) + make() + make("install") + + def setup_dependent_environment(self, module, spec, dependent_spec): + # TODO treat OS that are not Linux... + lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' + + spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib + spec['scalapack'].cc_link = spec['scalapack'].fc_link + spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, + 'libscalapack%s' % lib_suffix)] From 974fc65e0f15ac87d2422858d71e0c7624fb7726 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 14:38:55 +0100 Subject: [PATCH 037/339] renaming packages to follow changes on develop --- var/spack/{ => repos/builtin}/packages/mumps/Makefile.inc | 0 var/spack/{ => repos/builtin}/packages/mumps/package.py | 0 .../{ => repos/builtin}/packages/netlib-scalapack/package.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/mumps/Makefile.inc (100%) rename var/spack/{ => repos/builtin}/packages/mumps/package.py (100%) rename var/spack/{ => repos/builtin}/packages/netlib-scalapack/package.py (100%) diff --git a/var/spack/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc similarity index 100% rename from var/spack/packages/mumps/Makefile.inc rename to var/spack/repos/builtin/packages/mumps/Makefile.inc diff --git a/var/spack/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py similarity index 100% rename from var/spack/packages/mumps/package.py rename to var/spack/repos/builtin/packages/mumps/package.py diff --git a/var/spack/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py similarity index 100% rename from var/spack/packages/netlib-scalapack/package.py rename to var/spack/repos/builtin/packages/netlib-scalapack/package.py From 19caac69d8b3c8383c7b59c2b39ec3bf54406fa8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 20 Jan 2016 15:22:49 +0100 Subject: [PATCH 038/339] openmpi : turned torque support into a variant (default false) hdf5 : fixed a few bugs, removed suspicious macro deinition, etc. --- .../repos/builtin/packages/hdf5/package.py | 84 +++++++++++++++---- .../repos/builtin/packages/openmpi/package.py | 24 +++--- 2 files changed, 79 insertions(+), 29 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 9a401643411..ac78d8e9617 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -1,5 +1,31 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * + class Hdf5(Package): """HDF5 is a data model, library, and file format for storing and managing data. It supports an unlimited variety of datatypes, and is designed for @@ -7,7 +33,7 @@ class Hdf5(Package): """ homepage = "http://www.hdfgroup.org/HDF5/" - url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" + url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 @@ -15,26 +41,53 @@ class Hdf5(Package): version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') + variant('debug', default=False, description='Builds a debug version of the library') + variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') + variant('unsupported', default=False, description='Enables unsupported configuration options') + variant('mpi', default=False, description='Enable MPI support') - variant('threadsafe', default=False, description='Enable multithreading') + variant('threadsafe', default=False, description='Enable thread-safe capabilities') depends_on("mpi", when='+mpi') depends_on("zlib") - # TODO: currently hard-coded to use OpenMPI + def validate(self, spec): + """ + Checks if incompatible variants have been activated at the same time + + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + if '+fortran' in spec and not self.compiler.fc: + msg = 'cannot build a fortran variant without a fortran compiler' + raise RuntimeError(msg) + + if '+threadsafe' in spec and ('+cxx' in spec or '+fortran' in spec): + raise RuntimeError("cannot use variant +threadsafe with either +cxx or +fortran") + def install(self, spec, prefix): + self.validate(spec) + # Handle compilation after spec validation extra_args = [] + if '+debug' in spec: + extra_args.append('--enable-debug=all') + else: + extra_args.append('--enable-production') + + if '+unsupported' in spec: + extra_args.append("--enable-unsupported") + if '+cxx' in spec: - extra_args.extend([ - '--enable-cxx' - ]) + extra_args.append('--enable-cxx') + if '+fortran' in spec: extra_args.extend([ '--enable-fortran', '--enable-fortran2003' ]) + if '+mpi' in spec: # The HDF5 configure script warns if cxx and mpi are enabled # together. There doesn't seem to be a real reason for this, except @@ -43,27 +96,26 @@ def install(self, spec, prefix): # this is not actually a problem. extra_args.extend([ "--enable-parallel", - "--enable-unsupported", "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", - "FC=%s" % spec['mpi'].prefix.bin + "/mpifort", ]) - if '+threads' in spec: - if '+cxx' in spec or '+fortran' in spec: - die("Cannot use variant +threads with either +cxx or +fortran") + + if '+cxx' in spec: + extra_args.append("CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") + + if '+fortran' in spec: + extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + + if '+threadsafe' in spec: extra_args.extend([ '--enable-threadsafe', '--disable-hl', - 'CPPFLAGS=-DHDatexit=""', - 'CFLAGS=-DHDatexit=""' ]) configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-shared", + "--enable-shared", # TODO : this should be enabled by default, remove it? *extra_args) - make() make("install") diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 463719f9dbe..1b7f1c2011d 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -19,43 +19,44 @@ class Openmpi(Package): version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') - version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') - version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') + version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') + version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") patch('configure.patch', when="@1.10.0:") - variant('psm', default=False, description='Build support for the PSM library.') + variant('psm', default=False, description='Build support for the PSM library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + # TODO : variant support for other schedulers is missing + variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') + provides('mpi@:2.2', when='@1.6.5') provides('mpi@:3.0', when='@1.7.5:') - depends_on('hwloc') - def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' + os.environ['OMPI_CC'] = 'cc' os.environ['OMPI_CXX'] = 'c++' os.environ['OMPI_FC'] = 'f90' os.environ['OMPI_F77'] = 'f77' - def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, - "--with-tm", # necessary for Torque support "--enable-shared", "--enable-static"] # Variants + if '+tm' in spec: + config_args.append("--with-tm") # necessary for Torque support + if '+psm' in spec: config_args.append("--with-psm") @@ -85,7 +86,6 @@ def install(self, spec, prefix): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -94,7 +94,7 @@ def filter_compilers(self): to Spack's generic cc, c++ and f90. We want them to be bound to whatever compiler they were built with. """ - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : False } + kwargs = {'ignore_absent': True, 'backup': False, 'string': False} dir = os.path.join(self.prefix, 'share/openmpi/') cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', @@ -132,5 +132,3 @@ def filter_compilers(self): if not os.path.islink(path): filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, path, **kwargs) - - From b71b478a3626ae9e1a0d3b5c7ab9945b5c478200 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 20 Jan 2016 18:31:02 +0100 Subject: [PATCH 039/339] Minor correction to install mumps+mpi --- .../repos/builtin/packages/mumps/Makefile.inc | 3 -- .../repos/builtin/packages/mumps/package.py | 32 +++++++++++++++---- .../packages/netlib-scalapack/package.py | 17 +++++++--- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc index 4b6696db5c3..2e6a041878c 100644 --- a/var/spack/repos/builtin/packages/mumps/Makefile.inc +++ b/var/spack/repos/builtin/packages/mumps/Makefile.inc @@ -23,9 +23,6 @@ LIBPAR = $(SCALAP) LIBOTHERS = -lpthread -#Preprocessor defs for calling Fortran from C (-DAdd_ or -DAdd__ or -DUPPER) -CDEFS = -DAdd_ - #Sequential: ifeq ($(MUMPS_TYPE),seq) INCS = $(INCSEQ) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index b4e38c02de4..44a37903cc8 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -15,11 +15,10 @@ class Mumps(Package): variant('ptscotch', default=False, description='Activate PT-Scotch as a possible ordering library') variant('metis', default=False, description='Activate Metis as a possible ordering library') variant('parmetis', default=False, description='Activate Parmetis as a possible ordering library') - variant('double', default=True, description='Activate dmumps') - variant('float', default=True, description='Activate smumps') - variant('complex', default=True, description='Activate cmumps and/or zmumps') + variant('double', default=True, description='Activate the compilation of dmumps') + variant('float', default=True, description='Activate the compilation of smumps') + variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') - variant('double', default=False, description='Use double precision floating point types') depends_on('scotch + esmumps', when='~ptscotch+scotch') @@ -27,10 +26,16 @@ class Mumps(Package): depends_on('metis', when='~parmetis+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') + depends_on('lapack') depends_on('scalapack', when='+mpi') depends_on('mpi', when='+mpi') - - def patch(self): + + # this function is not a patch function because in case scalapack + # is needed it uses self.spec['scalapack'].fc_link set by the + # setup_dependent_environment in scalapck. This happen after patch + # end before install + # def patch(self): + def write_makefile_inc(self): if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') @@ -61,8 +66,13 @@ def patch(self): makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + # TODO: test this part, it needs a full blas, scalapack and + # partitionning environment with 64bit integers if '+idx64' in self.spec: makefile_conf.extend( + # the fortran compilation flags most probably are + # working only for intel and gnu compilers this is + # perhaps something the compiler should provide ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', 'OPTL = -O ', 'OPTC = -O -DINTSIZE64']) @@ -78,7 +88,8 @@ def patch(self): ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), - "SCALAP = %s" % self.spec['scalapack'].fc_link]) + "SCALAP = %s" % self.spec['scalapack'].fc_link, + "MUMPS_TYPE = par"]) else: makefile_conf.extend( ["CC = cc", @@ -86,6 +97,11 @@ def patch(self): "FL = fc", "MUMPS_TYPE = seq"]) + # TODO: change the value to the correct one according to the + # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER + makefile_conf.append("CDEFS = -DAdd_") + + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') with open(makefile_inc_template, "r") as fh: @@ -113,6 +129,8 @@ def install(self, spec, prefix): if '+complex' in spec: make_libs.append('zexamples') + self.write_makefile_inc() + make(*make_libs) install_tree('lib', prefix.lib) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 69df4d9da75..5be91c4a40c 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -9,23 +9,30 @@ class NetlibScalapack(Package): version('2.0.2', '2f75e600a2ba155ed9ce974a1c4b536f') version('2.0.1', '17b8cde589ea0423afe1ec43e7499161') version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') - + # versions before 2.0.0 are not using cmake and requires blacs as + # a separated package + variant('shared', default=True, description='Build the shared library version') - + variant('fpic', default=False, description="Build with -fpic compiler option") + provides('scalapack') depends_on('mpi') - depends_on('blas') depends_on('lapack') def install(self, spec, prefix): options = [ "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', - "-DCMAKE_C_FLAGS=-fPIC", - "-DCMAKE_Fortran_FLAGS=-fPIC", + "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) ] + if '+fpic' in spec: + options.extend([ + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_Fortran_FLAGS=-fPIC" + ]) + options.extend(std_cmake_args) with working_dir('spack-build', create=True): From e25150296a4ef34a80a664565e3d26f00f7d999f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 14:44:43 -0600 Subject: [PATCH 040/339] Redirect STDERR to STDOUT for compiler version This is necessary for the NAG Fortran compiler, which prints its version message to STDERR instead of STDOUT. This message was previously being ignored, preventing spack from finding the compiler's version or automatically adding it to the compilers.yaml configuration file. --- lib/spack/spack/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a665f6062d2..887e416dc58 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,6 +24,7 @@ ############################################################################## import os import re +import subprocess import itertools from datetime import datetime @@ -51,7 +52,7 @@ def _verify_executables(*paths): def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=os.devnull) + output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' From 088b6f0ea9953a3f007d207d06ca3150f83b3be4 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 17:19:01 -0600 Subject: [PATCH 041/339] Fixed typo when running spack help --- lib/spack/spack/cmd/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 39c225e9b26..307ee8982d5 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -32,7 +32,7 @@ import spack import spack.cmd -description="Print out locations of various diectories used by Spack" +description="Print out locations of various directories used by Spack" def setup_parser(subparser): global directories From 785c01218ffb33b085c786f19b81ebbefcb3d0c6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 21 Jan 2016 10:06:43 +0100 Subject: [PATCH 042/339] elpa : added missing dependencies --- var/spack/repos/builtin/packages/elpa/package.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py index 6825056a04b..2ade5b0b37b 100644 --- a/var/spack/repos/builtin/packages/elpa/package.py +++ b/var/spack/repos/builtin/packages/elpa/package.py @@ -36,11 +36,20 @@ class Elpa(Package): version('2015.11.001', 'de0f35b7ee7c971fd0dca35c900b87e6', url='http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz') + variant('openmp', default=False, description='Activates OpenMP support') + depends_on('mpi') depends_on('blas') depends_on('lapack') + depends_on('scalapack') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + + options = ["--prefix=%s" % prefix] + + if '+openmp' in spec: + options.append("--enable-openmp") + + configure(*options) make() make("install") From 950246b331c74700e01dc48a86f84bf47d528af3 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 09:54:24 -0600 Subject: [PATCH 043/339] Remove constraint on dependency version --- var/spack/repos/builtin/packages/hdf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 7882665dbaf..43b59802301 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -12,7 +12,7 @@ class Hdf(Package): version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') depends_on("jpeg") - depends_on("szip@2.1") + depends_on("szip") depends_on("zlib") From ea9d0c0263126c370119e5fc435652c7214e658a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 09:58:47 -0600 Subject: [PATCH 044/339] Remove constraints on dependency versions --- var/spack/repos/builtin/packages/mpc/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py index 65c0ba7ea13..108fec678f1 100644 --- a/var/spack/repos/builtin/packages/mpc/package.py +++ b/var/spack/repos/builtin/packages/mpc/package.py @@ -34,8 +34,8 @@ class Mpc(Package): version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - depends_on("gmp@4.3.2:") - depends_on("mpfr@2.4.2:") + depends_on("gmp") + depends_on("mpfr") def url_for_version(self, version): if version < Version("1.0.1"): From 03d5ee6738e49c4402cffe7503ea92c25edf9720 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 10:02:33 -0600 Subject: [PATCH 045/339] Remove constraint on dependency version --- var/spack/repos/builtin/packages/mpfr/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py index 2758b0da2ed..a1bd7529cfe 100644 --- a/var/spack/repos/builtin/packages/mpfr/package.py +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -33,7 +33,7 @@ class Mpfr(Package): version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') - depends_on('gmp@4.1.0:') + depends_on('gmp') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 10848c2e9a7cd3771c469dfd5ddb235f76b229ba Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 21 Jan 2016 02:40:38 -0800 Subject: [PATCH 046/339] Refactor args for Executable.__call__ - simplify output, error, and input redirection - `return_output=True` is now `output=str` - `return_output=True` will still work for the time being but is deprecated. - previously you could say `return_output=True` and `output=`, which wouldn't actually write to the stream. Now you actually can't specify erroneous cases since there is only one parameter with mutually exclusive options.. --- lib/spack/spack/cmd/bootstrap.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/compiler.py | 3 +- lib/spack/spack/fetch_strategy.py | 6 +- lib/spack/spack/test/cc.py | 6 +- lib/spack/spack/test/make_executable.py | 48 +++++----- lib/spack/spack/test/mock_repo.py | 4 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/util/executable.py | 95 +++++++++++++++---- .../repos/builtin/packages/gcc/package.py | 6 +- 11 files changed, 116 insertions(+), 60 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index e4ec7da35db..bdbd623b39a 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,7 +42,7 @@ def get_origin_url(): git = which('git', required=True) origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - return_output=True) + output=str) return origin_url.strip() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7cea39cb55a..edcea0718cd 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -132,7 +132,7 @@ def __call__(self, stage): # Peek inside the tarball. tar = which('tar') output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) + "--exclude=*/*/*", "-tf", stage.archive_file, output=str) lines = output.split("\n") # Set the configure line to the one that matched. diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 448f7628415..cf478d3763e 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -79,7 +79,7 @@ def list_packages(rev): git = get_git() relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, - return_output=True) + output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 887e416dc58..12c02e0ea2b 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,7 +24,6 @@ ############################################################################## import os import re -import subprocess import itertools from datetime import datetime @@ -52,7 +51,7 @@ def _verify_executables(*paths): def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) + output = compiler(version_arg, output=str, error=str) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0657146bf63..337dd1e1988 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -154,7 +154,7 @@ def fetch(self): # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -375,7 +375,7 @@ def __init__(self, **kwargs): @property def git_version(self): - vstring = self.git('--version', return_output=True).lstrip('git version ') + vstring = self.git('--version', output=str).lstrip('git version ') return Version(vstring) @@ -518,7 +518,7 @@ def fetch(self): def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" - status = self.svn('status', '--no-ignore', return_output=True) + status = self.svn('status', '--no-ignore', output=str) self.svn('status', '--no-ignore') for line in status.split('\n'): if not re.match('^[I?]', line): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 4188b8d5505..905af28a06d 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -65,17 +65,17 @@ def setUp(self): def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cc(*args, return_output=True).strip(), expected) + self.assertEqual(self.cc(*args, output=str).strip(), expected) def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.ld(*args, return_output=True).strip(), expected) + self.assertEqual(self.ld(*args, output=str).strip(), expected) def check_cpp(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cpp(*args, return_output=True).strip(), expected) + self.assertEqual(self.cpp(*args, output=str).strip(), expected) def test_vcheck_mode(self): diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 09efec85805..d568a28d443 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -56,47 +56,47 @@ def tearDown(self): def test_make_normal(self): make = MakeExecutable('make', 8) - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') def test_make_explicit(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') def test_make_one_job(self): make = MakeExecutable('make', 1) - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') def test_make_parallel_false(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=False, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=False, output=str).strip(), '') + self.assertEqual(make('install', parallel=False, output=str).strip(), 'install') def test_make_parallel_disabled(self): make = MakeExecutable('make', 8) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] @@ -106,20 +106,20 @@ def test_make_parallel_precedence(self): # These should work os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index c454b1f106c..9738ba4e728 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -141,7 +141,7 @@ def __init__(self): self.url = self.path def rev_hash(self, rev): - return git('rev-parse', rev, return_output=True).strip() + return git('rev-parse', rev, output=str).strip() class MockSvnRepo(MockVCSRepo): @@ -193,4 +193,4 @@ def __init__(self): def get_rev(self): """Get current mercurial revision.""" - return hg('id', '-i', return_output=True).strip() + return hg('id', '-i', output=str).strip() diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 2ee4748fdbd..7d150b42f41 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -65,7 +65,7 @@ def tearDown(self): def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" def get_rev(): - output = svn('info', return_output=True) + output = svn('info', output=str) self.assertTrue("Revision" in output) for line in output.split('\n'): match = re.match(r'Revision: (\d+)', line) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index ba765eb6629..fc27b789d06 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -55,24 +55,80 @@ def command(self): def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - ignore_errors = kwargs.get("ignore_errors", ()) + """Run this executable in a subprocess. + + Arguments + args + command line arguments to the executable to run. + + Optional arguments + + fail_on_error + + Raise an exception if the subprocess returns an + error. Default is True. When not set, the return code is + avaiale as `exe.returncode`. + + ignore_errors + + An optional list/tuple of error codes that can be + *ignored*. i.e., if these codes are returned, this will + not raise an exception when `fail_on_error` is `True`. + + output, error + + These arguments allow you to specify new stdout and stderr + values. They default to `None`, which means the + subprocess will inherit the parent's file descriptors. + + You can set these to: + - python streams, e.g. open Python file objects, or os.devnull; + - filenames, which will be automatically opened for writing; or + - `str`, as in the Python string type. If you set these to `str`, + output and error will be written to pipes and returned as + a string. If both `output` and `error` are set to `str`, + then one string is returned containing output concatenated + with error. + + input + + Same as output, error, but `str` is not an allowed value. + + Deprecated arguments + + return_output[=False] + + Setting this to True is the same as setting output=str. + This argument may be removed in future Spack versions. + + """ + fail_on_error = kwargs.pop("fail_on_error", True) + ignore_errors = kwargs.pop("ignore_errors", ()) + + # TODO: This is deprecated. Remove in a future version. + return_output = kwargs.pop("return_output", False) # Default values of None says to keep parent's file descriptors. - output = kwargs.get("output", None) - error = kwargs.get("error", None) - input = kwargs.get("input", None) + if return_output: + output = str + else: + output = kwargs.pop("output", None) + + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) + if input is str: + raise ValueError("Cannot use `str` as input stream.") def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True + elif arg is str: + return subprocess.PIPE, False else: return arg, False - output, ostream = streamify(output, 'w') - error, estream = streamify(error, 'w') - input, istream = streamify(input, 'r') + ostream, close_ostream = streamify(output, 'w') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -92,19 +148,20 @@ def streamify(arg, mode): tty.debug(cmd_line) try: - if return_output: - output = subprocess.PIPE - proc = subprocess.Popen( - cmd, stdin=input, stderr=error, stdout=output) + cmd, stdin=istream, stderr=estream, stdout=ostream) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + + if output is str or error is str: + result = '' + if output is str: result += out + if error is str: result += err + return result except OSError, e: raise ProcessError( @@ -119,9 +176,9 @@ def streamify(arg, mode): % (proc.returncode, cmd_line)) finally: - if ostream: output.close() - if estream: error.close() - if istream: input.close() + if close_ostream: output.close() + if close_estream: error.close() + if close_istream: input.close() def __eq__(self, other): diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 61b16f3fd8d..3e5895cfb83 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -50,7 +50,7 @@ class Gcc(Package): version('4.5.4', '27e459c2566b8209ab064570e1b378f7') variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") - + depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: @@ -102,7 +102,7 @@ def install(self, spec, prefix): configure(*options) make() make("install") - + self.write_rpath_specs() @@ -121,7 +121,7 @@ def write_rpath_specs(self): return gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + lines = gcc('-dumpspecs', output=str).strip().split("\n") specs_file = join_path(self.spec_dir, 'specs') with closing(open(specs_file, 'w')) as out: for line in lines: From 6b3f023d98c7d34ab859e53b2e0953f529325f6e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 21 Jan 2016 14:05:16 -0500 Subject: [PATCH 047/339] Update OpenMPI to 1.10.2 --- var/spack/repos/builtin/packages/openmpi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 1b7f1c2011d..9d57f49427b 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -17,6 +17,7 @@ class Openmpi(Package): list_url = "http://www.open-mpi.org/software/ompi/" list_depth = 3 + version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c') version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') From 2f70c842b8bc7f897b5a18df10c72ae6a7277fbd Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 21 Jan 2016 14:13:39 -0500 Subject: [PATCH 048/339] Disable patch that was merged into upstream in 1.10.2 --- var/spack/repos/builtin/packages/openmpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 9d57f49427b..e4484af8c5e 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -25,7 +25,7 @@ class Openmpi(Package): patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") - patch('configure.patch', when="@1.10.0:") + patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') From ca3edc36738f561c3a27369835da80152b5f9f78 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 13:20:39 -0600 Subject: [PATCH 049/339] Add symlink cxx to point to cc --- lib/spack/env/cxx | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/spack/env/cxx diff --git a/lib/spack/env/cxx b/lib/spack/env/cxx new file mode 120000 index 00000000000..2652f5f42c0 --- /dev/null +++ b/lib/spack/env/cxx @@ -0,0 +1 @@ +cc \ No newline at end of file From 571386cdf4c6249958008d3a3f0bd24192fd111a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 13:55:59 -0600 Subject: [PATCH 050/339] Remove cxx symlink, tell NAG to look for c++ symlink --- lib/spack/env/cxx | 1 - lib/spack/spack/compilers/nag.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 lib/spack/env/cxx diff --git a/lib/spack/env/cxx b/lib/spack/env/cxx deleted file mode 120000 index 2652f5f42c0..00000000000 --- a/lib/spack/env/cxx +++ /dev/null @@ -1 +0,0 @@ -cc \ No newline at end of file diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index f1cc6be0d5c..527a05a090c 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -16,7 +16,7 @@ class Nag(Compiler): # Named wrapper links within spack.build_env_path link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml 'cc' : 'cc', - 'cxx' : 'cxx', + 'cxx' : 'c++', 'f77' : 'nag/nagfor', 'fc' : 'nag/nagfor' } From 2e58bc31138ee172ea55ad652aaf49b09fe6e135 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 21 Jan 2016 19:57:49 -0800 Subject: [PATCH 051/339] Sticking with additive approach but now most libraries are installed by default. --- .../repos/builtin/packages/boost/package.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 1992d4d39a5..f31bc445b8d 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -45,28 +45,36 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - libs = ['chrono', + default_install_libs = set(['chrono', 'date_time', 'filesystem', - 'iostreams', + 'graph', + 'iostreams', + 'log', + 'math', 'random', 'regex', 'serialization', 'signals', 'system', 'thread', - 'wave', - 'mpi', - 'python'] + 'wave']) - for lib in libs: - variant(lib, default=False, description="Compile with {0} library" - .format(lib)) + # These are not installed by default because they pull in many dependencies + # and/or because there is a great deal of customization possible (and it + # would be difficult or tedious to choose sensible defaults here). + default_noinstall_libs = set(['mpi', 'python']) + + all_libs = default_install_libs | default_noinstall_libs + + for lib in all_libs: + variant(lib, default=(lib in default_install_libs), + description="Compile with {0} library".format(lib)) variant('debug', default=False, description='Switch to the debug version of Boost') variant('shared', default=True, description="Additionally build shared libraries") variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") - variant('singlethreaded', default=False, description="Build single-threaded versions of libraries") + variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") depends_on('icu', when='+regex_icu') From 4a55b97d11739a0ee6ca5ddcbdf35826ca292469 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 12:37:12 -0800 Subject: [PATCH 052/339] Fixed reference --- var/spack/repos/builtin/packages/boost/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index f31bc445b8d..467e9a61c65 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -163,7 +163,7 @@ def determine_b2_options(self, spec, options): def install(self, spec, prefix): withLibs = list() - for lib in Boost.libs: + for lib in Boost.all_libs: if "+{0}".format(lib) in spec: withLibs.append(lib) if not withLibs: From 10de5a3ec76fbf1244c43a843ae26a67fc879ddb Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 22 Jan 2016 13:02:18 -0800 Subject: [PATCH 053/339] Patch fix for boost@1.60.0%gcc@4.4.7 --- .../builtin/packages/boost/boost_11856.patch | 34 +++++++++++++++++++ .../repos/builtin/packages/boost/package.py | 3 ++ 2 files changed, 37 insertions(+) create mode 100644 var/spack/repos/builtin/packages/boost/boost_11856.patch diff --git a/var/spack/repos/builtin/packages/boost/boost_11856.patch b/var/spack/repos/builtin/packages/boost/boost_11856.patch new file mode 100644 index 00000000000..3b4052ca187 --- /dev/null +++ b/var/spack/repos/builtin/packages/boost/boost_11856.patch @@ -0,0 +1,34 @@ +--- a/libs/container/src/pool_resource.cpp 2015-11-06 12:49:55.000000000 -0800 ++++ b/libs/container/src/pool_resource.cpp 2015-12-22 07:54:36.202131121 -0800 +@@ -32,11 +32,11 @@ + class pool_data_t + : public block_slist_base<> + { +- typedef block_slist_base<> block_slist_base; ++ typedef block_slist_base<> block_slist_base_t; + + public: + explicit pool_data_t(std::size_t initial_blocks_per_chunk) +- : block_slist_base(), next_blocks_per_chunk(initial_blocks_per_chunk) ++ : block_slist_base_t(), next_blocks_per_chunk(initial_blocks_per_chunk) + { slist_algo::init_header(&free_slist); } + + void *allocate_block() BOOST_NOEXCEPT +@@ -59,7 +59,7 @@ + void release(memory_resource &upstream) + { + slist_algo::init_header(&free_slist); +- this->block_slist_base::release(upstream); ++ this->block_slist_base_t::release(upstream); + next_blocks_per_chunk = pool_options_minimum_max_blocks_per_chunk; + } + +@@ -72,7 +72,7 @@ + + //Minimum block size is at least max_align, so all pools allocate sizes that are multiple of max_align, + //meaning that all blocks are max_align-aligned. +- char *p = static_cast(block_slist_base::allocate(blocks_per_chunk*pool_block, mr)); ++ char *p = static_cast(block_slist_base_t::allocate(blocks_per_chunk*pool_block, mr)); + + //Create header types. This is no-throw + for(std::size_t i = 0, max = blocks_per_chunk; i != max; ++i){ diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 3427b74ad68..e3fb516be9e 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -54,6 +54,9 @@ class Boost(Package): depends_on('bzip2', when='+compression') depends_on('zlib', when='+compression') + # Patch fix from https://svn.boost.org/trac/boost/ticket/11856 + patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] From 9f99ee61c733e8fee8ae4058fb9198288af40fc6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 13:25:45 -0800 Subject: [PATCH 054/339] 1. added default install libs (atomic, test, locale, program_options) 2. clarify comment for default_noinstall_libs 3. renamed regex_icu variant to icu_support (both the locale and regex libs can use it) 4. explicitly set b2 install ICU_PATH when regex_icu is activated --- .../repos/builtin/packages/boost/package.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 467e9a61c65..1403ea44118 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -45,24 +45,28 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - default_install_libs = set(['chrono', + default_install_libs = set(['atomic', + 'chrono', 'date_time', 'filesystem', 'graph', 'iostreams', + 'locale', 'log', 'math', + 'program_options', 'random', 'regex', 'serialization', 'signals', 'system', + 'test', 'thread', 'wave']) - # These are not installed by default because they pull in many dependencies - # and/or because there is a great deal of customization possible (and it - # would be difficult or tedious to choose sensible defaults here). + # mpi/python are not installed by default because they pull in many + # dependencies and/or because there is a great deal of customization + # possible (and it would be difficult to choose sensible defaults) default_noinstall_libs = set(['mpi', 'python']) all_libs = default_install_libs | default_noinstall_libs @@ -75,9 +79,9 @@ class Boost(Package): variant('shared', default=True, description="Additionally build shared libraries") variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") - variant('regex_icu', default=False, description="Include regex ICU support (by default false even if regex library is compiled)") + variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") - depends_on('icu', when='+regex_icu') + depends_on('icu', when='+icu_support') depends_on('python', when='+python') depends_on('mpi', when='+mpi') depends_on('bzip2', when='+iostreams') @@ -134,6 +138,9 @@ def determine_b2_options(self, spec, options): else: options.append('variant=release') + if '+icu_support' in spec: + options.extend(['-s', 'ICU_PATH=%s' % spec['icu'].prefix]) + if '+iostreams' in spec: options.extend([ '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, From a653d2f5e2f296ff66eb41ffb9f2f6e9b9bdb3b5 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 22 Jan 2016 13:43:16 -0800 Subject: [PATCH 055/339] Slightly more robust approach for setting defaults for noinstall_libs --- var/spack/repos/builtin/packages/boost/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 1403ea44118..a30cd7cc35b 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -72,7 +72,7 @@ class Boost(Package): all_libs = default_install_libs | default_noinstall_libs for lib in all_libs: - variant(lib, default=(lib in default_install_libs), + variant(lib, default=(lib not in default_noinstall_libs), description="Compile with {0} library".format(lib)) variant('debug', default=False, description='Switch to the debug version of Boost') From 64a954922504d08d2e109ae93be06c5095243b67 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 15:31:07 -0800 Subject: [PATCH 056/339] Minor cleanup. 80 char width for command description. --- lib/spack/spack/cmd/test-install.py | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 74017f59fb3..656873a2f03 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -37,20 +37,20 @@ from spack.fetch_strategy import FetchError import spack.cmd -description = "Treat package installations as unit tests and output formatted test results" +description = "Run package installation as a unit test, output formatted results." def setup_parser(subparser): subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") - + subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") - + subparser.add_argument( '-o', '--output', action='store', help="test output goes in this file") - + subparser.add_argument( 'package', nargs=argparse.REMAINDER, help="spec of package to install") @@ -59,10 +59,10 @@ class JunitResultFormat(object): def __init__(self): self.root = ET.Element('testsuite') self.tests = [] - + def add_test(self, buildId, testResult, buildInfo=None): self.tests.append((buildId, testResult, buildInfo)) - + def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) for buildId, testResult, buildInfo in self.tests: @@ -84,25 +84,25 @@ class TestResult(object): PASSED = 0 FAILED = 1 SKIPPED = 2 - + class BuildId(object): def __init__(self, spec): self.name = spec.name self.version = spec.version self.hashId = spec.dag_hash() - + def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) def __hash__(self): return hash((self.name, self.version, self.hashId)) - + def __eq__(self, other): if not isinstance(other, BuildId): return False - - return ((self.name, self.version, self.hashId) == + + return ((self.name, self.version, self.hashId) == (other.name, other.version, other.hashId)) @@ -114,12 +114,12 @@ def fetch_log(path): def failed_dependencies(spec): - return set(childSpec for childSpec in spec.dependencies.itervalues() if not + return set(childSpec for childSpec in spec.dependencies.itervalues() if not spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): - # Post-order traversal is not strictly required but it makes sense to output + # Post-order traversal is not strictly required but it makes sense to output # tests for dependencies first. for spec in topSpec.traverse(order='post'): if spec not in newInstalls: @@ -143,12 +143,12 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): re.search('error:', line, re.IGNORECASE)) errOutput = errMessages if errMessages else lines[-10:] errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, + [spec.to_yaml(), "Errors:"], errOutput, ["Build Log:", package.build_log_path])) else: result = TestResult.PASSED errOutput = None - + bId = BuildId(spec) output.add_test(bId, result, errOutput) @@ -163,18 +163,18 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - + specs = spack.cmd.parse_specs(args.package, concretize=True) if len(specs) > 1: tty.die("Only 1 top-level package can be specified") topSpec = iter(specs).next() - + newInstalls = set() for spec in topSpec.traverse(): package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) - + if not args.output: bId = BuildId(topSpec) outputDir = join_path(os.getcwd(), "test-output") @@ -183,7 +183,7 @@ def test_install(parser, args): outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) else: outputFpath = args.output - + for spec in topSpec.traverse(order='post'): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which @@ -202,7 +202,7 @@ def test_install(parser, args): pass except FetchError: pass - + jrf = JunitResultFormat() handled = {} create_test_output(topSpec, newInstalls, jrf) From be354e85c9d168b5764f8fec6cd1da53a5b4a7fe Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 16:16:43 -0800 Subject: [PATCH 057/339] Better errors for mkdirp failure in mirror. --- lib/spack/spack/mirror.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef25..d4f1bb89fec 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -147,7 +147,11 @@ def create(path, specs, **kwargs): # Get the absolute path of the root before we start jumping around. mirror_root = os.path.abspath(path) if not os.path.isdir(mirror_root): - mkdirp(mirror_root) + try: + mkdirp(mirror_root) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. present = [] @@ -164,7 +168,11 @@ def create(path, specs, **kwargs): # create a subdirectory for the current package@version archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) - mkdirp(subdir) + try: + mkdirp(subdir) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % subdir, str(e)) if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) From fe50593c66d8cf7ee527fa13928990595d8b0a29 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 20:14:21 -0800 Subject: [PATCH 058/339] Minor line width reductions. --- lib/spack/spack/cmd/mirror.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 885483a8406..92b92e07af6 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -53,11 +53,13 @@ def setup_parser(subparser): create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") create_parser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + 'specs', nargs=argparse.REMAINDER, + help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") create_parser.add_argument( - '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', + help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -74,7 +76,8 @@ def setup_parser(subparser): help="Configuration scope to modify.") # Remove - remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) + remove_parser = sp.add_parser('remove', aliases=['rm'], + help=mirror_remove.__doc__) remove_parser.add_argument('name') remove_parser.add_argument( '--scope', choices=scopes, default=spack.cmd.default_modify_scope, From d3ff8ca00f7f47d365ea88ed717fa7940a1346c6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 24 Jan 2016 20:14:40 -0800 Subject: [PATCH 059/339] Fixes #382: Issues with spack fetch. - urljoin() was compliant with RFC 1808 but not with my understanding of how paths should be joined. - updated path joining logic to comply. --- lib/spack/spack/stage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 31635a854a5..19b7d03664d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -252,7 +252,13 @@ def fetch(self): self.skip_checksum_for_mirror = True if self.mirror_path: mirrors = spack.config.get_config('mirrors') - urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] + + # Join URLs of mirror roots with mirror paths. Because + # urljoin() will strip everything past the final '/' in + # the root, so we add a '/' if it is not present. + mirror_roots = [root if root.endswith('/') else root + '/' + for root in mirrors.values()] + urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that From a48d0a494f8524c789e94a09c182ff1b6e465b8c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 00:32:14 -0800 Subject: [PATCH 060/339] Refactor mock_repo: add a destroy() method. - classes using mock_repo need not know about its stage now. --- lib/spack/spack/test/git_fetch.py | 5 +---- lib/spack/spack/test/hg_fetch.py | 5 +---- lib/spack/spack/test/install.py | 4 +--- lib/spack/spack/test/mirror.py | 23 ++++++++++------------- lib/spack/spack/test/mock_repo.py | 6 ++++++ lib/spack/spack/test/svn_fetch.py | 5 +---- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 3813079065b..d84433176a8 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -56,10 +56,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index ee8327aec84..bbcb64e4c1a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -53,10 +53,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 628329a4237..c09bd24c8e1 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -59,9 +59,7 @@ def setUp(self): def tearDown(self): super(InstallTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() + self.repo.destroy() # Turn checksumming back on spack.do_checksum = True diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 04e9e3db2ef..0093b5d82f1 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -44,8 +44,16 @@ def setUp(self): self.repos = {} + def tearDown(self): + """Destroy all the stages created by the repos in setup.""" + super(MirrorTest, self).tearDown() + for repo in self.repos.values(): + repo.destroy() + self.repos.clear() + + def set_up_package(self, name, MockRepoClass, url_attr): - """Use this to set up a mock package to be mirrored. + """Set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. 2. Point the package's version args at that repo. @@ -65,17 +73,6 @@ def set_up_package(self, name, MockRepoClass, url_attr): pkg.versions[v][url_attr] = repo.url - def tearDown(self): - """Destroy all the stages created by the repos in setup.""" - super(MirrorTest, self).tearDown() - - for name, repo in self.repos.items(): - if repo.stage: - pass #repo.stage.destroy() - - self.repos.clear() - - def check_mirror(self): stage = Stage('spack-mirror-test') mirror_root = join_path(stage.path, 'test-mirror') @@ -129,7 +126,7 @@ def check_mirror(self): self.assertTrue(all(l in exclude for l in dcmp.left_only)) finally: - pass #stage.destroy() + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index 9738ba4e728..ed94023b0eb 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -55,6 +55,12 @@ def __init__(self, stage_name, repo_name): mkdirp(self.path) + def destroy(self): + """Destroy resources associated with this mock repo.""" + if self.stage: + self.stage.destroy() + + class MockArchive(MockRepo): """Creates a very simple archive directory with a configure script and a makefile that installs to a prefix. Tars it up into an archive.""" diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 7d150b42f41..454a7f1d1ff 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -55,10 +55,7 @@ def setUp(self): def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() From 5502fd1054e665d5deb028217f554640bedf8514 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 01:59:02 -0800 Subject: [PATCH 061/339] More thorough mirror test: test full round-trip - Old test: did not attempt to actually fetch mirrored packages. - New test: 1. Creates a temporary mirror, 2. Registers it with spack, 3. Fetches from it, and 4. Verifies that the fetched archive matches the original - This test will hopefully mean that `spack mirror` is less brittle from now on. --- lib/spack/spack/cmd/mirror.py | 1 + lib/spack/spack/package.py | 14 ++++------ lib/spack/spack/stage.py | 10 ++++--- lib/spack/spack/test/mirror.py | 49 +++++++++++++++++----------------- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 92b92e07af6..a71ad16b658 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -172,6 +172,7 @@ def mirror_create(args): specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: new_specs = set() for spec in specs: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fba2912b757..8cb947c276c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -631,7 +631,7 @@ def remove_prefix(self): spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self): + def do_fetch(self, mirror_only=False): """Creates a stage directory and downloads the taball for this package. Working directory will be set to the stage directory. """ @@ -656,7 +656,7 @@ def do_fetch(self): raise FetchError( "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) - self.stage.fetch() + self.stage.fetch(mirror_only) ########## # Fetch resources @@ -677,7 +677,8 @@ def do_fetch(self): if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: @@ -691,8 +692,7 @@ def _expand_archive(stage, name=self.name): else: tty.msg("Already staged %s in %s." % (name, stage.path)) - - self.do_fetch() + self.do_fetch(mirror_only) _expand_archive(self.stage) ########## @@ -835,10 +835,6 @@ def _resource_stage(self, resource): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def _build_logger(self, log_path): - """Create a context manager to log build output.""" - - def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 19b7d03664d..79c9030e20f 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -97,7 +97,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') - self.tmp_root = find_tmp_root() self.path = None @@ -240,11 +239,13 @@ def chdir(self): tty.die("Setup failed: no such directory: " + self.path) - def fetch(self): + def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.default_fetcher] + fetchers = [] + if not mirror_only: + fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a @@ -267,10 +268,11 @@ def fetch(self): if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest - # Have to skip the checkesum for things archived from + # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) + # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 0093b5d82f1..046ec56604c 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -77,6 +77,10 @@ def check_mirror(self): stage = Stage('spack-mirror-test') mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( @@ -85,7 +89,7 @@ def check_mirror(self): # Stage directory exists self.assertTrue(os.path.isdir(mirror_root)) - # subdirs for each package + # check that there are subdirs for each package for name in self.repos: subdir = join_path(mirror_root, name) self.assertTrue(os.path.isdir(subdir)) @@ -93,38 +97,35 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Decompress archive in the mirror - archive = files[0] - archive_path = join_path(subdir, archive) - decomp = decompressor_for(archive_path) + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - with working_dir(subdir): - decomp(archive_path) - - # Find the untarred archive directory. - files = os.listdir(subdir) - self.assertEqual(len(files), 2) - self.assertTrue(archive in files) - files.remove(archive) - - expanded_archive = join_path(subdir, files[0]) - self.assertTrue(os.path.isdir(expanded_archive)) + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) # Compare the original repo with the expanded archive - repo = self.repos[name] - if not 'svn' in name: - original_path = repo.path - else: - co = 'checked_out' - svn('checkout', repo.url, co) - original_path = join_path(subdir, co) + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, expanded_archive) + dcmp = dircmp(original_path, pkg.stage.source_path) # make sure there are no new files in the expanded tarball self.assertFalse(dcmp.right_only) + + # and that all original files are present. self.assertTrue(all(l in exclude for l in dcmp.left_only)) + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() finally: stage.destroy() From 41cd8f8e6d78f45c31145652a518b9d6d7d9a9c3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 02:57:23 -0800 Subject: [PATCH 062/339] Fixes #74: minor fixes to spack mirror. --- lib/spack/spack/cmd/mirror.py | 4 +++- lib/spack/spack/mirror.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index a71ad16b658..8e9438c1a37 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -144,15 +144,17 @@ def mirror_list(args): def _read_specs_from_file(filename): + specs = [] with open(filename, "r") as stream: for i, string in enumerate(stream): try: s = Spec(string) s.package - args.specs.append(s) + specs.append(s) except SpackError, e: tty.die("Parse error in %s, line %d:" % (args.file, i+1), ">>> " + string, str(e)) + return specs def mirror_create(args): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index d4f1bb89fec..341cc4cb885 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -78,6 +78,7 @@ def get_matching_versions(specs, **kwargs): continue num_versions = kwargs.get('num_versions', 0) + matching_spec = [] for i, v in enumerate(reversed(sorted(pkg.versions))): # Generate no more than num_versions versions for each spec. if num_versions and i >= num_versions: @@ -88,7 +89,11 @@ def get_matching_versions(specs, **kwargs): s = Spec(pkg.name) s.versions = VersionList([v]) s.variants = spec.variants.copy() - matching.append(s) + matching_spec.append(s) + + if not matching_spec: + tty.warn("No known version matches spec: %s" % spec) + matching.extend(matching_spec) return matching From fb5274145aac9266435f3e16f19ce3e30ba1550f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 21 Jan 2016 11:06:27 -0600 Subject: [PATCH 063/339] Fix regex to find version on OS X --- lib/spack/spack/compilers/clang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 340051019cc..998d4e59323 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -54,4 +54,4 @@ def default_version(self, comp): Thread model: posix """ return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM) ([^ )]+)') + comp, '--version', r'(?:clang version|based on LLVM|LLVM version) ([^ )]+)') From 004c99ab2ffa0a9e4635f3dbf9a9beaceb6727d3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 25 Jan 2016 03:41:28 -0800 Subject: [PATCH 064/339] Fixes #259: Apple clang compiler detection - Clang compilers are now versioned with the raw LLVM version or, on macs, with Apple's version with a suffix. --- lib/spack/spack/compilers/clang.py | 32 +++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 998d4e59323..e406d86a245 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -22,7 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import re +import spack.compiler as cpr from spack.compiler import * +from spack.util.executable import * class Clang(Compiler): # Subclasses use possible names of C compiler @@ -47,11 +50,34 @@ class Clang(Compiler): @classmethod def default_version(self, comp): """The '--version' option works for clang compilers. - Output looks like this:: + On most platforms, output looks like this:: clang version 3.1 (trunk 149096) Target: x86_64-unknown-linux-gnu Thread model: posix + + On Mac OS X, it looks like this: + + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix + """ - return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM|LLVM version) ([^ )]+)') + if comp not in cpr._version_cache: + compiler = Executable(comp) + output = compiler('--version', output=str, error=str) + + ver = 'unknown' + match = re.search(r'^Apple LLVM version ([^ )]+)', output) + if match: + # Apple's LLVM compiler has its own versions, so suffix them. + ver = match.group(1) + '-apple' + else: + # Normal clang compiler versions are left as-is + match = re.search(r'^clang version ([^ )]+)', output) + if match: + ver = match.group(1) + + cpr._version_cache[comp] = ver + + return cpr._version_cache[comp] From 323bba99f18d0d774e76feb7f0d28a87fac2d93f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 25 Jan 2016 09:29:51 -0600 Subject: [PATCH 065/339] Fix variable name bug --- bin/spack | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/spack b/bin/spack index 93263217ac1..31165bba9d1 100755 --- a/bin/spack +++ b/bin/spack @@ -42,9 +42,9 @@ SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) import warnings -# Avoid warnings when nose is installed with the python exe being used to run -# spack. Note this must be done after Spack's external libs directory is added -# to sys.path. +# Avoid warnings when nose is installed with the python exe being used to run +# spack. Note this must be done after Spack's external libs directory is added +# to sys.path. with warnings.catch_warnings(): warnings.filterwarnings("ignore", ".*nose was already imported") import nose @@ -62,7 +62,7 @@ for pyc_file in orphaned_pyc_files: try: os.remove(pyc_file) except OSError as e: - print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc + print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc_file # If there is no working directory, use the spack prefix. try: From 1d18f2031c086b4f1e5bb52744c75b232cbf2e7f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 25 Jan 2016 14:57:33 -0600 Subject: [PATCH 066/339] Add szip variant --- var/spack/repos/builtin/packages/hdf5/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index ac78d8e9617..5321a191f0e 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -48,9 +48,11 @@ class Hdf5(Package): variant('unsupported', default=False, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') + variant('szip', default=False, description='Enable szip support') variant('threadsafe', default=False, description='Enable thread-safe capabilities') depends_on("mpi", when='+mpi') + depends_on("szip", when='+szip') depends_on("zlib") def validate(self, spec): @@ -105,6 +107,9 @@ def install(self, spec, prefix): if '+fortran' in spec: extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + if '+szip' in spec: + extra_args.append("--with-szlib=%s" % spec['szip'].prefix) + if '+threadsafe' in spec: extra_args.extend([ '--enable-threadsafe', From 093b8317998c395ed3e2dee7f19d1bdcea1b0560 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 12:08:28 +0100 Subject: [PATCH 067/339] Fixes issues introduced after merge with conflicts --- lib/spack/spack/package.py | 3 +++ lib/spack/spack/stage.py | 43 +++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 06aecf11bd5..14cbc0dbce6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -804,6 +804,9 @@ def _get_resources(self): for when_spec, resource_list in self.resources.items(): if when_spec in self.spec: resources.extend(resource_list) + # Sorts the resources by the length of the string representing their destination. Since any nested resource + # must contain another resource's name in its path, it seems that should work + resources = sorted(resources, key=lambda res: len(res.destination)) return resources def _resource_stage(self, resource): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 1deac1137c4..6ba301d95f4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,4 +1,4 @@ -############################################################################## +1 ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import re +import errno import shutil import tempfile from urlparse import urljoin @@ -38,7 +38,6 @@ import spack.fetch_strategy as fs import spack.error - STAGE_PREFIX = 'spack-stage-' @@ -95,7 +94,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs): raise ValueError("Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -104,7 +103,6 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.path = None self._setup() - def _cleanup_dead_links(self): """Remove any dead links in the stage directory.""" for file in os.listdir(spack.stage_path): @@ -114,7 +112,6 @@ def _cleanup_dead_links(self): if not os.path.exists(path): os.unlink(path) - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -132,7 +129,7 @@ def _need_to_create_path(self): # Path looks ok, but need to check the target of the link. if os.path.islink(self.path): real_path = os.path.realpath(self.path) - real_tmp = os.path.realpath(self.tmp_root) + real_tmp = os.path.realpath(self.tmp_root) if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the right spot, @@ -151,7 +148,6 @@ def _need_to_create_path(self): return False - def _setup(self): """Creates the stage directory. If spack.use_tmp_stage is False, the stage directory is created @@ -200,7 +196,6 @@ def _setup(self): # Make sure we can actually do something with the stage we made. ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -217,7 +212,6 @@ def archive_file(self): else: return None - @property def source_path(self): """Returns the path to the expanded/checked out source code @@ -232,7 +226,6 @@ def source_path(self): return p return None - def chdir(self): """Changes directory to the stage path. Or dies if it is not set up.""" if os.path.isdir(self.path): @@ -240,7 +233,6 @@ def chdir(self): else: tty.die("Setup failed: no such directory: " + self.path) - def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() @@ -293,7 +285,6 @@ def fetch(self, mirror_only=False): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -307,7 +298,6 @@ def check(self): else: self.fetcher.check() - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -320,7 +310,6 @@ def expand_archive(self): else: tty.msg("Already staged %s in %s." % (self.name, self.path)) - def chdir_to_source(self): """Changes directory to the expanded archive directory. Dies with an error if there was no expanded archive. @@ -333,14 +322,12 @@ def chdir_to_source(self): if not os.listdir(path): tty.die("Archive was empty for %s" % self.name) - def restage(self): """Removes the expanded archive path if it exists, then re-expands the archive. """ self.fetcher.reset() - def destroy(self): """Remove this stage directory.""" remove_linked_tree(self.path) @@ -367,11 +354,24 @@ def expand_archive(self): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - link_path = join_path(root_stage.source_path, resource.destination, value) + target_path = join_path(root_stage.source_path, resource.destination) + destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) - if not os.path.exists(link_path): + + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: + raise + + if not os.path.exists(destination_path): # Create a symlink - os.symlink(source_path, link_path) + tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) + shutil.move(source_path, destination_path) @pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) @@ -380,6 +380,7 @@ class StageComposite: Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it. """ + @property def source_path(self): return self[0].source_path @@ -394,6 +395,7 @@ def chdir_to_source(self): class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" + def __init__(self, path): self.archive_file = None self.path = path @@ -431,7 +433,6 @@ def _get_mirrors(): return [val for name, val in config.iteritems()] - def ensure_access(file=spack.stage_path): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): From 5e3b7a424765ee64f71192bdca129c104a1246f6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 14:11:23 +0100 Subject: [PATCH 068/339] Fixes issues introduced after merge with conflicts --- lib/spack/spack/package.py | 2 +- lib/spack/spack/stage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 14cbc0dbce6..a1b8d12ec29 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -702,7 +702,7 @@ def do_stage(self, mirror_only=False): if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - self.do_fetch() + self.do_fetch(mirror_only) self.stage.expand_archive() self.stage.chdir_to_source() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 6ba301d95f4..f217450d42d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -1,4 +1,4 @@ -1 ############################################################################## +############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # From ee6f69a227a4ee005759a64bdaa542745972d8e0 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 26 Jan 2016 14:25:16 +0100 Subject: [PATCH 069/339] Unit tests : hack to make them pass while a decision on how to proceed is made --- lib/spack/spack/test/mirror.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 046ec56604c..f83cc8090c9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -102,6 +102,7 @@ def check_mirror(self): spec = Spec(name).concretized() pkg = spec.package + pkg._stage = None saved_checksum_setting = spack.do_checksum try: # Stage the archive from the mirror and cd to it. From 7891346a4ff939bf14c1c3a688f50a2be9b5dbbe Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 26 Jan 2016 09:52:01 -0800 Subject: [PATCH 070/339] fix for issue 398, added extensions_for in RepoPath --- lib/spack/spack/repository.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 31596cee7a5..b5df1168b66 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -233,6 +233,11 @@ def providers_for(self, vpkg_spec): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. From c6bb00085f89b6db37a03674ba09714adab1f2f0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 26 Jan 2016 12:04:48 -0600 Subject: [PATCH 071/339] Extensive modifications to NetCDF package --- .../repos/builtin/packages/netcdf/package.py | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 239644d894c..93c4410146f 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -2,27 +2,53 @@ class Netcdf(Package): """NetCDF is a set of software libraries and self-describing, machine-independent - data formats that support the creation, access, and sharing of array-oriented - scientific data.""" + data formats that support the creation, access, and sharing of array-oriented + scientific data.""" homepage = "http://www.unidata.ucar.edu/software/netcdf/" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" + version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + variant('fortran', default=False, description="Download and install NetCDF-Fortran") + variant('hdf4', default=False, description="Enable HDF4 support") + patch('netcdf-4.3.3-mpi.patch') # Dependencies: - depends_on("cmake @2.8.12:") - # >HDF5 - depends_on("hdf5") + depends_on("curl") # required for DAP support + depends_on("hdf", when='+hdf4') + depends_on("hdf5") # required for NetCDF-4 support + depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % prefix, - "-DENABLE_DAP:BOOL=OFF", # Disable DAP. - "-DBUILD_SHARED_LIBS:BOOL=OFF") # Don't build shared libraries (use static libs). + config_args = [ + "--enable-fsync", + "--enable-v2", + "--enable-utilities", + "--enable-shared", + "--enable-static", + "--enable-largefile", + # necessary for HDF5 support + "--enable-netcdf-4", + "--enable-dynamic-loading", + # necessary for DAP support + "--enable-dap" + ] - make() - make("install") + # HDF4 support + if '+hdf4' in spec: + config_args.append("--enable-hdf4") + + # Fortran support + # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. + # They can be installed separately, but this bootstrap procedure + # should be able to install both at the same time. + # Note: this is a new experimental feature + if '+fortran' in spec: + config_args.append("--enable-remote-fortran-bootstrap") + + configure(*config_args) + make() + make("install") From 81ccba202c601edacaf483eacc7373d1947e4875 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 26 Jan 2016 15:38:22 -0600 Subject: [PATCH 072/339] Make szip a variant --- .../repos/builtin/packages/hdf/package.py | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 43b59802301..1ecb1671836 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -11,8 +11,10 @@ class Hdf(Package): version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') + variant('szip', default=False, description="Enable szip support") + depends_on("jpeg") - depends_on("szip") + depends_on("szip", when='+szip') depends_on("zlib") @@ -21,15 +23,22 @@ def url_for_version(self, version): def install(self, spec, prefix): - configure('--prefix=%s' % prefix, - '--with-jpeg=%s' % spec['jpeg'].prefix, - '--with-szlib=%s' % spec['szip'].prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--disable-netcdf', - '--enable-fortran', - '--disable-shared', - '--enable-static', - '--enable-production') + config_args = [ + '--prefix=%s' % prefix, + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', + '--enable-fortran', + '--disable-shared', + '--enable-static', + '--enable-production' + ] + + # SZip support + if '+szip' in spec: + config_args.append('--with-szlib=%s' % spec['szip'].prefix) + + configure(*config_args) make() make("install") From 23af31cb1c773cd5a01f93a4896d794ddf908bae Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 26 Jan 2016 17:55:32 -0800 Subject: [PATCH 073/339] adding the cmocka unit testing library --- .../repos/builtin/packages/cmocka/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cmocka/package.py diff --git a/var/spack/repos/builtin/packages/cmocka/package.py b/var/spack/repos/builtin/packages/cmocka/package.py new file mode 100644 index 00000000000..7377016a6b0 --- /dev/null +++ b/var/spack/repos/builtin/packages/cmocka/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Cmocka(Package): + """Unit-testing framework in pure C""" + homepage = "https://cmocka.org/" + url = "https://cmocka.org/files/1.0/cmocka-1.0.1.tar.xz" + + version('1.0.1', 'ed861e501a21a92b2af63e466df2015e') + parallel = False + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + + make() + make("install") From f1d8f30342f21d772b2bf4fc76a9a8b262e479c5 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 27 Jan 2016 10:54:41 +0100 Subject: [PATCH 074/339] Bug fix: precedence in % operator seems to change depending in the python version --- var/spack/repos/builtin/packages/netlib-scalapack/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 5be91c4a40c..22d538560e8 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -22,8 +22,8 @@ class NetlibScalapack(Package): def install(self, spec, prefix): options = [ - "-DBUILD_SHARED_LIBS:BOOL=%s" % 'ON' if '+shared' in spec else 'OFF', - "-DBUILD_STATIC_LIBS:BOOL=%s" % 'OFF' if '+shared' in spec else 'ON', + "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), + "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) ] From f7f192e12b0387537652ba4914e6e484d29ef728 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 27 Jan 2016 17:12:24 +0100 Subject: [PATCH 075/339] Added unit tests for util.pattern --- lib/spack/spack/test/__init__.py | 1 + lib/spack/spack/test/pattern.py | 101 +++++++++++++++++++++++++++++++ lib/spack/spack/util/pattern.py | 2 +- 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/test/pattern.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index a569cbbf359..4b9a361d4bf 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -48,6 +48,7 @@ 'package_sanity', 'config', 'directory_layout', + 'pattern', 'python_version', 'git_fetch', 'svn_fetch', diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py new file mode 100644 index 00000000000..64fc9187f9a --- /dev/null +++ b/lib/spack/spack/test/pattern.py @@ -0,0 +1,101 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import unittest + +import spack.util.pattern as pattern + + +class CompositeTest(unittest.TestCase): + + def setUp(self): + class Base: + counter = 0 + + def add(self): + raise NotImplemented('add not implemented') + + def subtract(self): + raise NotImplemented('subtract not implemented') + + class One(Base): + def add(self): + Base.counter += 1 + + def subtract(self): + Base.counter -= 1 + + class Two(Base): + def add(self): + Base.counter += 2 + + def subtract(self): + Base.counter -= 2 + + self.Base = Base + self.One = One + self.Two = Two + + def test_composite_from_method_list(self): + + @pattern.composite(method_list=['add', 'subtract']) + class CompositeFromMethodList: + pass + + composite = CompositeFromMethodList() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_composite_from_interface(self): + + @pattern.composite(interface=self.Base) + class CompositeFromInterface: + pass + + composite = CompositeFromInterface() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_error_conditions(self): + + with self.assertRaises(TypeError): + @pattern.composite(interface=self.Base, container=2) + class CompositeFromInterface: + pass + + with self.assertRaises(TypeError): + @pattern.composite() + class CompositeFromInterface: + pass diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index 73c1e26aa5d..17a126498bf 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -53,7 +53,7 @@ def composite(interface=None, method_list=None, container=list): raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite") def cls_decorator(cls): - # Retrieve the base class of the composite. Inspect its the methods and decide which ones will be overridden + # Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden def no_special_no_private(x): return inspect.ismethod(x) and not x.__name__.startswith('_') From 47035671e80f071c970dc66b15ad7ced2b87329d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 27 Jan 2016 17:22:12 +0100 Subject: [PATCH 076/339] unit tests : now compliant with 2.6 --- lib/spack/spack/test/pattern.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py index 64fc9187f9a..6c783c6a5f7 100644 --- a/lib/spack/spack/test/pattern.py +++ b/lib/spack/spack/test/pattern.py @@ -90,12 +90,15 @@ class CompositeFromInterface: def test_error_conditions(self): - with self.assertRaises(TypeError): + def wrong_container(): @pattern.composite(interface=self.Base, container=2) class CompositeFromInterface: pass - with self.assertRaises(TypeError): + def no_methods(): @pattern.composite() class CompositeFromInterface: pass + + self.assertRaises(TypeError, wrong_container) + self.assertRaises(TypeError, no_methods) From 6122642b818a6f1cfa999482949fed7154b2dbf0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 27 Jan 2016 16:16:33 -0600 Subject: [PATCH 077/339] More NetCDF changes --- .../repos/builtin/packages/hdf/package.py | 9 ++--- .../packages/netcdf/netcdf-4.3.3-mpi.patch | 25 -------------- .../repos/builtin/packages/netcdf/package.py | 34 ++++++++++++++++--- 3 files changed, 35 insertions(+), 33 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py index 1ecb1671836..ac6435f2a24 100644 --- a/var/spack/repos/builtin/packages/hdf/package.py +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -24,12 +24,13 @@ def url_for_version(self, version): def install(self, spec, prefix): config_args = [ + 'CFLAGS=-fPIC', '--prefix=%s' % prefix, - '--with-jpeg=%s' % spec['jpeg'].prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--disable-netcdf', + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', # must be disabled to build NetCDF with HDF4 support '--enable-fortran', - '--disable-shared', + '--disable-shared', # fortran and shared libraries are not compatible '--enable-static', '--enable-production' ] diff --git a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch deleted file mode 100644 index 46dda5fc9de..00000000000 --- a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff -Nur netcdf-4.3.3/CMakeLists.txt netcdf-4.3.3.mpi/CMakeLists.txt ---- netcdf-4.3.3/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/CMakeLists.txt 2015-10-14 16:44:41.176300658 -0400 -@@ -753,6 +753,7 @@ - SET(USE_PARALLEL OFF CACHE BOOL "") - MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.") - ELSE() -+ FIND_PACKAGE(MPI REQUIRED) - SET(USE_PARALLEL ON CACHE BOOL "") - SET(STATUS_PARALLEL "ON") - ENDIF() -diff -Nur netcdf-4.3.3/liblib/CMakeLists.txt netcdf-4.3.3.mpi/liblib/CMakeLists.txt ---- netcdf-4.3.3/liblib/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/liblib/CMakeLists.txt 2015-10-14 16:44:57.757793634 -0400 -@@ -71,6 +71,10 @@ - SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY}) - ENDIF() - -+IF(USE_PARALLEL) -+ SET(TLL_LIBS ${TLL_LIBS} ${MPI_C_LIBRARIES}) -+ENDIF() -+ - IF(USE_HDF4) - SET(TLL_LIBS ${TLL_LIBS} ${HDF4_LIBRARIES}) - ENDIF() diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 93c4410146f..3cd0b2ee7a8 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -12,9 +12,7 @@ class Netcdf(Package): version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('fortran', default=False, description="Download and install NetCDF-Fortran") - variant('hdf4', default=False, description="Enable HDF4 support") - - patch('netcdf-4.3.3-mpi.patch') + variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: depends_on("curl") # required for DAP support @@ -23,7 +21,13 @@ class Netcdf(Package): depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): + # Environment variables + CPPFLAGS = [] + LDFLAGS = [] + LIBS = [] + config_args = [ + "--prefix=%s" % prefix, "--enable-fsync", "--enable-v2", "--enable-utilities", @@ -37,18 +41,40 @@ def install(self, spec, prefix): "--enable-dap" ] + CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) + # HDF4 support + # As of NetCDF 4.1.3, "--with-hdf4=..." is no longer a valid option + # You must use the environment variables CPPFLAGS and LDFLAGS if '+hdf4' in spec: config_args.append("--enable-hdf4") + CPPFLAGS.append("-I%s/include" % spec['hdf'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf'].prefix) + LIBS.append( "-l%s" % "jpeg") + + if 'szip' in spec: + CPPFLAGS.append("-I%s/include" % spec['szip'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['szip'].prefix) + LIBS.append( "-l%s" % "sz") # Fortran support # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. # They can be installed separately, but this bootstrap procedure # should be able to install both at the same time. - # Note: this is a new experimental feature + # Note: this is a new experimental feature. if '+fortran' in spec: config_args.append("--enable-remote-fortran-bootstrap") + config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) + config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) + config_args.append('LIBS=%s' % ' '.join(LIBS)) + configure(*config_args) make() make("install") + + # After installing NetCDF-C, install NetCDF-Fortran + if '+fortran' in spec: + make("build-netcdf-fortran") + make("install-netcdf-fortran") From 07bb6fef01bfe48aa22c39e53b75e4c779ac0c2e Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 10:58:56 +0100 Subject: [PATCH 078/339] resource directive : now works with all the fetch strategies available --- lib/spack/spack/directives.py | 2 +- lib/spack/spack/fetch_strategy.py | 19 ++++++++++++++++--- lib/spack/spack/package.py | 1 - .../repos/builtin/packages/llvm/package.py | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 0b98211cb9e..5745adce63c 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -296,8 +296,8 @@ def resource(pkg, **kwargs): raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) - fetcher = from_kwargs(**kwargs) name = kwargs.get('name') + fetcher = from_kwargs(**kwargs) resources.append(Resource(name, fetcher, destination, placement)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b2ff587a606..83a2dbb59c5 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -44,6 +44,7 @@ import sys import re import shutil +import copy from functools import wraps import llnl.util.tty as tty from llnl.util.filesystem import * @@ -370,8 +371,12 @@ class GitFetchStrategy(VCSFetchStrategy): required_attributes = ('git',) def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **kwargs) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None @property @@ -479,8 +484,12 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **kwargs) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @@ -556,8 +565,12 @@ class HgFetchStrategy(VCSFetchStrategy): required_attributes = ['hg'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **kwargs) + 'hg', 'revision', **forwarded_args) self._hg = None @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a1b8d12ec29..8019b29cba5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -435,7 +435,6 @@ def url_for_version(self, version): def _make_resource_stage(self, root_stage, fetcher, resource): resource_stage_folder = self._resource_stage(resource) - # FIXME : works only for URLFetchStrategy resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, name=resource_stage_folder, mirror_path=resource_mirror) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a2b2c6ecccb..1805d3ded80 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -171,6 +171,25 @@ class Llvm(Package): when='@%(version)s' % release, placement=resources[name].get('placement', None)) + # SVN - current develop + version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') + resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk', + destination='tools', when='@develop', placement='clang') + resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk', + destination='projects', when='@develop', placement='compiler-rt') + resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk', + destination='projects', when='@develop', placement='openmp') + resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk', + destination='projects', when='@develop', placement='libcxx') + resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk', + destination='projects', when='@develop', placement='libcxxabi') + resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk', + destination='tools', when='@develop', placement='polly') + resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk', + destination='tools', when='@develop', placement='lldb') + + + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] From 603f5e69ee43e1249e413c966cde3e02ba10a1a3 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 11:30:28 +0100 Subject: [PATCH 079/339] removed outdated code : os detection (will be handled by platforms) and openssl modifications --- lib/spack/spack/architecture.py | 89 ------------------- .../repos/builtin/packages/openssl/package.py | 15 ---- 2 files changed, 104 deletions(-) delete mode 100644 lib/spack/spack/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py deleted file mode 100644 index 2701fab90c9..00000000000 --- a/lib/spack/spack/architecture.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import re -import platform - -from llnl.util.lang import memoized - -import spack -import spack.error as serr - - -class InvalidSysTypeError(serr.SpackError): - def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) - - -class NoSysTypeError(serr.SpackError): - def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() - else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') - - -def get_sys_type_from_platform(): - """Return the architecture from Python's platform module.""" - sys_type = platform.system() + '-' + platform.machine() - sys_type = re.sub(r'[^\w-]', '_', sys_type) - return sys_type.lower() - - -@memoized -def sys_type(): - """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_sys_type_from_platform] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - - return sys_type diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 8f0427796be..bbb169ec6bc 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,7 +1,5 @@ from spack import * -import spack.util.architecture as arch - class Openssl(Package): """The OpenSSL Project is a collaborative effort to develop a robust, commercial-grade, full-featured, and Open Source @@ -40,16 +38,3 @@ def install(self, spec, prefix): make() make("install") - - @when(arch.os_is_in('darwin')) - def install(self, spec, prefix): - perl = which('perl') - perl("./Configure", - "--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared", - "darwin64-x86_64-cc") - filter_file(r'-arch x86_64', '', 'Makefile') - From 0cf03518f3edf8b70331b1b5f2eae171d0a0dccf Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 28 Jan 2016 12:06:30 +0100 Subject: [PATCH 080/339] reverted deletion of the wrong `architecture.py` --- lib/spack/spack/architecture.py | 89 ++++++++++++++++++++++++++++ lib/spack/spack/util/architecture.py | 46 -------------- 2 files changed, 89 insertions(+), 46 deletions(-) create mode 100644 lib/spack/spack/architecture.py delete mode 100644 lib/spack/spack/util/architecture.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py new file mode 100644 index 00000000000..2701fab90c9 --- /dev/null +++ b/lib/spack/spack/architecture.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import re +import platform + +from llnl.util.lang import memoized + +import spack +import spack.error as serr + + +class InvalidSysTypeError(serr.SpackError): + def __init__(self, sys_type): + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) + + +class NoSysTypeError(serr.SpackError): + def __init__(self): + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() + else: + return spack.sys_type + + +def get_sys_type_from_environment(): + """Return $SYS_TYPE or None if it's not defined.""" + return os.environ.get('SYS_TYPE') + + +def get_sys_type_from_platform(): + """Return the architecture from Python's platform module.""" + sys_type = platform.system() + '-' + platform.machine() + sys_type = re.sub(r'[^\w-]', '_', sys_type) + return sys_type.lower() + + +@memoized +def sys_type(): + """Returns a SysType for the current machine.""" + methods = [get_sys_type_from_spack_globals, + get_sys_type_from_environment, + get_sys_type_from_platform] + + # search for a method that doesn't return None + sys_type = None + for method in methods: + sys_type = method() + if sys_type: break + + # Couldn't determine the sys_type for this machine. + if sys_type is None: + return "unknown_arch" + + if not isinstance(sys_type, basestring): + raise InvalidSysTypeError(sys_type) + + return sys_type diff --git a/lib/spack/spack/util/architecture.py b/lib/spack/spack/util/architecture.py deleted file mode 100644 index a020c74a7ac..00000000000 --- a/lib/spack/spack/util/architecture.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import sys - - -def os_is_in(*args): - """ - Return True if any element in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return any(map(lambda x: x == sys.platform, args)) - - -def os_is_not_in(*args): - """ - Return True if none of the elements in the list is equal to sys.platform, False otherwise - - :param args: list of names to be checked - :return: True or False - """ - return not os_is_in(*args) From 5850d8530ef0570e508fb1ebd53428e98cc761a3 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Thu, 28 Jan 2016 14:22:28 +0100 Subject: [PATCH 081/339] Adding the stager to checksum any url that spack can handle --- lib/spack/spack/cmd/md5.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index ef1e4f34756..b8ffb5dec79 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -30,8 +30,9 @@ from llnl.util.filesystem import * import spack.util.crypto +from spack.stage import Stage, FailedDownloadError -description = "Calculate md5 checksums for files." +description = "Calculate md5 checksums for files/urls." def setup_parser(subparser): setup_parser.parser = subparser @@ -45,9 +46,20 @@ def md5(parser, args): for f in args.files: if not os.path.isfile(f): - tty.die("Not a file: %s" % f) - if not can_access(f): - tty.die("Cannot read file: %s" % f) + stage = Stage(f) + try: + stage.fetch() + checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + print "%s %s" % (checksum, f) + except FailedDownloadError, e: + tty.msg("Failed to fetch %s" % url) + continue - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + finally: + stage.destroy() + else: + if not can_access(f): + tty.die("Cannot read file: %s" % f) + + checksum = spack.util.crypto.checksum(hashlib.md5, f) + print "%s %s" % (checksum, f) From 824546d343917fb24c1b94af0eeaea79371efb46 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Thu, 28 Jan 2016 14:43:30 +0100 Subject: [PATCH 082/339] correcting a bug when url and files are mixed --- lib/spack/spack/cmd/md5.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index b8ffb5dec79..879ef9f7b74 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -23,8 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import hashlib import argparse +import hashlib + +from contextlib import contextmanager import llnl.util.tty as tty from llnl.util.filesystem import * @@ -34,6 +36,19 @@ description = "Calculate md5 checksums for files/urls." +@contextmanager +def stager(url): + _cwd = os.getcwd() + _stager = Stage(url) + try: + _stager.fetch() + yield _stager + except FailedDownloadError: + tty.msg("Failed to fetch %s" % url) + finally: + _stager.destroy() + os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored + def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, @@ -46,17 +61,9 @@ def md5(parser, args): for f in args.files: if not os.path.isfile(f): - stage = Stage(f) - try: - stage.fetch() + with stager(f) as stage: checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) print "%s %s" % (checksum, f) - except FailedDownloadError, e: - tty.msg("Failed to fetch %s" % url) - continue - - finally: - stage.destroy() else: if not can_access(f): tty.die("Cannot read file: %s" % f) From 06f3cc33baeb1c8a8e99b0e0e602f398b4d221bd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 28 Jan 2016 11:41:12 -0600 Subject: [PATCH 083/339] Modify url settings for fish package --- var/spack/repos/builtin/packages/fish/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py index 12255587057..b5a4a2d209e 100644 --- a/var/spack/repos/builtin/packages/fish/package.py +++ b/var/spack/repos/builtin/packages/fish/package.py @@ -7,7 +7,8 @@ class Fish(Package): homepage = "http://fishshell.com/" url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage + list_url = "http://fishshell.com/files/" + list_depth = 2 version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') From 0e52c30bb8fbe954bdb2f74353865317cac18b85 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 28 Jan 2016 10:42:46 -0800 Subject: [PATCH 084/339] added py-wheel package --- .../repos/builtin/packages/py-wheel/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-wheel/package.py diff --git a/var/spack/repos/builtin/packages/py-wheel/package.py b/var/spack/repos/builtin/packages/py-wheel/package.py new file mode 100644 index 00000000000..3118e745199 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wheel/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyWheel(Package): + """A built-package format for Python.""" + + homepage = "https://pypi.python.org/pypi/wheel" + url = "https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz" + + version('0.26.0', '4cfc6e7e3dc7377d0164914623922a10') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 7e65f4da82cb9176247c1aa9ab72f40116cc6a8e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 28 Jan 2016 13:22:56 -0800 Subject: [PATCH 085/339] Add the Caliper package --- .../repos/builtin/packages/caliper/package.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 var/spack/repos/builtin/packages/caliper/package.py diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py new file mode 100644 index 00000000000..b14a562aa8e --- /dev/null +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Caliper(Package): + """ + Caliper is a generic context annotation system. It gives programmers the + ability to provide arbitrary program context information to (performance) + tools at runtime. + """ + + homepage = "https://github.com/LLNL/Caliper" + url = "" + + version('master', git='ssh://git@cz-stash.llnl.gov:7999/piper/caliper.git') + + variant('mpi', default=False, description='Enable MPI function wrappers.') + + depends_on('libunwind') + depends_on('papi') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + with working_dir('build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") From 4f340315341a125bd01d0c844efc83454e531ca5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 28 Jan 2016 15:41:58 -0600 Subject: [PATCH 086/339] Add PnetCDF and M4 packages --- .../repos/builtin/packages/m4/package.py | 13 ++++++++++++ .../packages/parallel-netcdf/package.py | 20 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 var/spack/repos/builtin/packages/m4/package.py create mode 100644 var/spack/repos/builtin/packages/parallel-netcdf/package.py diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py new file mode 100644 index 00000000000..5d76d8866bc --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -0,0 +1,13 @@ +from spack import * + +class M4(Package): + """GNU M4 is an implementation of the traditional Unix macro processor.""" + homepage = "https://www.gnu.org/software/m4/m4.html" + url = "ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz" + + version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py new file mode 100644 index 00000000000..62a8f7ca0ba --- /dev/null +++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py @@ -0,0 +1,20 @@ +from spack import * + +class ParallelNetcdf(Package): + """Parallel netCDF (PnetCDF) is a library providing high-performance + parallel I/O while still maintaining file-format compatibility with + Unidata's NetCDF.""" + + homepage = "https://trac.mcs.anl.gov/projects/parallel-netcdf" + url = "http://cucis.ece.northwestern.edu/projects/PnetCDF/Release/parallel-netcdf-1.6.1.tar.gz" + + version('1.6.1', '62a094eb952f9d1e15f07d56e535052604f1ac34') + + depends_on("m4") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-mpi=%s" % spec['mpi'].prefix) + make() + make("install") From d9548c01afe555ee58019b2ef5cb8938cd71cee3 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 28 Jan 2016 15:47:37 -0700 Subject: [PATCH 087/339] Correct package URL --- var/spack/repos/builtin/packages/caliper/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index b14a562aa8e..d51b4a4dd57 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -10,7 +10,7 @@ class Caliper(Package): homepage = "https://github.com/LLNL/Caliper" url = "" - version('master', git='ssh://git@cz-stash.llnl.gov:7999/piper/caliper.git') + version('master', git='ssh://git@github.com:LLNL/Caliper.git') variant('mpi', default=False, description='Enable MPI function wrappers.') From 2bea7f8d69064f893a21cfc53f3d781b572e2f79 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Fri, 29 Jan 2016 00:29:20 -0500 Subject: [PATCH 088/339] Add the HPX-5 package. --- .../repos/builtin/packages/hpx/package.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/repos/builtin/packages/hpx/package.py diff --git a/var/spack/repos/builtin/packages/hpx/package.py b/var/spack/repos/builtin/packages/hpx/package.py new file mode 100644 index 00000000000..665d5c74073 --- /dev/null +++ b/var/spack/repos/builtin/packages/hpx/package.py @@ -0,0 +1,27 @@ +from spack import * +import os + +class Hpx(Package): + """The HPX-5 Runtime System. HPX-5 (High Performance ParalleX) is an + open source, portable, performance-oriented runtime developed at + CREST (Indiana University). HPX-5 provides a distributed + programming model allowing programs to run unmodified on systems + from a single SMP to large clusters and supercomputers with + thousands of nodes. HPX-5 supports a wide variety of Intel and ARM + platforms. It is being used by a broad range of scientific + applications enabling scientists to write code that performs and + scales better than contemporary runtimes.""" + homepage = "http://hpx.crest.iu.edu" + url = "http://hpx.crest.iu.edu/release/hpx-2.0.0.tar.gz" + + version('2.0.0', '3d2ff3aab6c46481f9ec65c5b2bfe7a6') + version('1.3.0', '2260ecc7f850e71a4d365a43017d8cee') + version('1.2.0', '4972005f85566af4afe8b71afbf1480f') + version('1.1.0', '646afb460ecb7e0eea713a634933ce4f') + version('1.0.0', '8020822adf6090bd59ed7fe465f6c6cb') + + def install(self, spec, prefix): + os.chdir("./hpx/") + configure('--prefix=%s' % prefix) + make() + make("install") From b574c4ad5190bad375b9b8b38857222d5e79682a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 29 Jan 2016 08:57:07 +0100 Subject: [PATCH 089/339] trilinos : updated package to meet changes in dependencies --- .../repos/builtin/packages/netcdf/package.py | 7 +++- .../builtin/packages/trilinos/package.py | 36 +++++++++++-------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 3cd0b2ee7a8..579282e7f76 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -11,13 +11,15 @@ class Netcdf(Package): version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + variant('mpi', default=True, description='Enables MPI parallelism') variant('fortran', default=False, description="Download and install NetCDF-Fortran") variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: depends_on("curl") # required for DAP support depends_on("hdf", when='+hdf4') - depends_on("hdf5") # required for NetCDF-4 support + depends_on("hdf5+mpi~cxx", when='+mpi') # required for NetCDF-4 support + depends_on("hdf5~mpi", when='~mpi') # required for NetCDF-4 support depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): @@ -41,6 +43,9 @@ def install(self, spec, prefix): "--enable-dap" ] + if '+mpi' in spec: + config_args.append('--enable-parallel') + CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 7c43f796a4e..e6b97022d92 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -10,6 +10,7 @@ class Trilinos(Package): homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') version('12.0.1', 'bd99741d047471e127b8296b2ec08017') version('11.14.3', '2f4f83f8333e4233c57d0f01c4b57426') @@ -17,33 +18,38 @@ class Trilinos(Package): version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') depends_on('boost') - depends_on('netcdf') depends_on('matio') depends_on('glm') depends_on('swig') + + # MPI related dependencies depends_on('mpi', when='+mpi') + depends_on('netcdf+mpi', when='+mpi') + depends_on('netcdf~mpi', when='~mpi') + + depends_on('python') # Needs py-numpy activated def install(self, spec, prefix): - - options = [ - '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DBUILD_SHARED_LIBS:BOOL=ON', - '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, - '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix - ] - if '+mpi' in spec: - mpi_options = ['-DTPL_ENABLE_MPI:BOOL=ON'] - options.extend(mpi_options) - - # -DCMAKE_INSTALL_PREFIX and all the likes... + options = [] options.extend(std_cmake_args) + + options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:BOOL=%s' % ('ON' if '+mpi' in spec else 'OFF'), + '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, + '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix + ]) + with working_dir('spack-build', create=True): cmake('..', *options) make() From cd547939138c00a45a2252f073ce21d3742d5dd0 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 29 Jan 2016 11:42:06 +0100 Subject: [PATCH 090/339] Cleaning up --- var/spack/repos/builtin/packages/openssl/package.py | 7 ++++--- var/spack/repos/builtin/packages/trilinos/package.py | 8 +++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index bbb169ec6bc..3b790eca66b 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -9,9 +9,10 @@ class Openssl(Package): homepage = "http://www.openssl.org" url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" - version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') - version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') - version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') # Not available + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') # Not available + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') # Not available + version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") parallel = False diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index e6b97022d92..edc40476e30 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -17,7 +17,6 @@ class Trilinos(Package): version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -30,9 +29,8 @@ class Trilinos(Package): depends_on('swig') # MPI related dependencies - depends_on('mpi', when='+mpi') - depends_on('netcdf+mpi', when='+mpi') - depends_on('netcdf~mpi', when='~mpi') + depends_on('mpi') + depends_on('netcdf+mpi') depends_on('python') # Needs py-numpy activated @@ -45,7 +43,7 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:BOOL=%s' % ('ON' if '+mpi' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:STRING=ON', '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix ]) From 74225544c6ae27e3ae07d37fe15935364c665f6e Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Fri, 29 Jan 2016 10:22:52 -0500 Subject: [PATCH 091/339] Rename the hpx package to hpx-5. --- .../builtin/packages/{hpx => hpx5}/package.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) rename var/spack/repos/builtin/packages/{hpx => hpx5}/package.py (57%) diff --git a/var/spack/repos/builtin/packages/hpx/package.py b/var/spack/repos/builtin/packages/hpx5/package.py similarity index 57% rename from var/spack/repos/builtin/packages/hpx/package.py rename to var/spack/repos/builtin/packages/hpx5/package.py index 665d5c74073..3dae3c4170c 100644 --- a/var/spack/repos/builtin/packages/hpx/package.py +++ b/var/spack/repos/builtin/packages/hpx5/package.py @@ -1,7 +1,7 @@ from spack import * import os -class Hpx(Package): +class Hpx5(Package): """The HPX-5 Runtime System. HPX-5 (High Performance ParalleX) is an open source, portable, performance-oriented runtime developed at CREST (Indiana University). HPX-5 provides a distributed @@ -20,8 +20,33 @@ class Hpx(Package): version('1.1.0', '646afb460ecb7e0eea713a634933ce4f') version('1.0.0', '8020822adf6090bd59ed7fe465f6c6cb') + variant('debug', default=False, description='Build a debug version of HPX-5') + variant('photon', default=False, description='Enable Photon support') + variant('mpi', default=False, description='Enable MPI support') + + depends_on("mpi", when='+mpi') + depends_on("mpi", when='+photon') + def install(self, spec, prefix): + extra_args = [] + if '+debug' in spec: + extra_args.extend([ + '--enable-debug', + 'CFLAGS=-g -O0' + ]) + else: + extra_args.append('CFLAGS=-O3') + + if '+mpi' in spec: + extra_args.append('--enable-mpi') + + if '+photon' in spec: + extra_args.extend([ + '--enable-mpi', + '--enable-photon' + ]) + os.chdir("./hpx/") - configure('--prefix=%s' % prefix) + configure('--prefix=%s' % prefix, *extra_args) make() make("install") From d14d50beb6dbb71c75a53902e70601c353496e09 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 29 Jan 2016 09:47:57 -0600 Subject: [PATCH 092/339] Checksum fix for hwloc --- var/spack/repos/builtin/packages/hwloc/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py index 60b315119ba..ab7205646ee 100644 --- a/var/spack/repos/builtin/packages/hwloc/package.py +++ b/var/spack/repos/builtin/packages/hwloc/package.py @@ -17,8 +17,8 @@ class Hwloc(Package): list_url = "http://www.open-mpi.org/software/hwloc/" list_depth = 3 - version('1.11.2', '486169cbe111cdea57be12638828ebbf') - version('1.11.1', '002742efd3a8431f98d6315365a2b543') + version('1.11.2', 'e4ca55c2a5c5656da4a4e37c8fc51b23') + version('1.11.1', 'feb4e416a1b25963ed565d8b42252fdc') version('1.9', '1f9f9155682fe8946a97c08896109508') depends_on('libpciaccess') From 3bf6fed7b3536d3b38a9855c2129956b24a77529 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 29 Jan 2016 11:29:27 -0800 Subject: [PATCH 093/339] updated openssl version --- var/spack/repos/builtin/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index bbb169ec6bc..05de35fca06 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -12,6 +12,7 @@ class Openssl(Package): version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') + version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") parallel = False From 360abb070a639711f740aa7ef73a84ab76aff14f Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 08:33:03 +0100 Subject: [PATCH 094/339] netcdf : fixed typo in configure option openssl : smarter URL computation --- .../repos/builtin/packages/netcdf/package.py | 3 +- .../repos/builtin/packages/openssl/package.py | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 579282e7f76..89b40f4a90d 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -1,5 +1,6 @@ from spack import * + class Netcdf(Package): """NetCDF is a set of software libraries and self-describing, machine-independent data formats that support the creation, access, and sharing of array-oriented @@ -44,7 +45,7 @@ def install(self, spec, prefix): ] if '+mpi' in spec: - config_args.append('--enable-parallel') + config_args.append('--enable-parallel4') CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 3b790eca66b..345fce0669e 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,3 +1,5 @@ +import urllib + from spack import * class Openssl(Package): @@ -9,9 +11,10 @@ class Openssl(Package): homepage = "http://www.openssl.org" url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" - version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') # Not available - version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') # Not available - version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') # Not available + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.1r', '1abd905e079542ccae948af37e393d28') + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') depends_on("zlib") @@ -39,3 +42,22 @@ def install(self, spec, prefix): make() make("install") + + def url_for_version(self, version): + # This URL is computed pinging the place where the latest version is stored. To avoid slowdown + # due to repeated pinging, we store the URL in a private attribute + openssl_url = getattr(self, '_openssl_url', None) + + if openssl_url is None: + latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' + older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' + # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then + # return the url for older format + version_number = '.'.join([str(x) for x in version[:-1]]) + older_url = older.format(version_number=version_number, version_full=version) + latest_url = latest.format(version=version) + response = urllib.urlopen(latest.format(version=version)) + openssl_url = older_url if response.getcode() == 404 else latest_url + self._openssl_url = openssl_url + + return openssl_url From e67507478a9f662d7adfa891bcb2c7b830b69f5c Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:55:39 +0100 Subject: [PATCH 095/339] opencv : reverted url_for_version modifications --- .../repos/builtin/packages/openssl/package.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 345fce0669e..36a33204fd8 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -42,22 +42,3 @@ def install(self, spec, prefix): make() make("install") - - def url_for_version(self, version): - # This URL is computed pinging the place where the latest version is stored. To avoid slowdown - # due to repeated pinging, we store the URL in a private attribute - openssl_url = getattr(self, '_openssl_url', None) - - if openssl_url is None: - latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' - older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' - # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then - # return the url for older format - version_number = '.'.join([str(x) for x in version[:-1]]) - older_url = older.format(version_number=version_number, version_full=version) - latest_url = latest.format(version=version) - response = urllib.urlopen(latest.format(version=version)) - openssl_url = older_url if response.getcode() == 404 else latest_url - self._openssl_url = openssl_url - - return openssl_url From c8d2275c06f05b8ddf2463c5aa110d1f44692038 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:56:48 +0100 Subject: [PATCH 096/339] opencv : reverted url_for_version modifications --- var/spack/repos/builtin/packages/openssl/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 36a33204fd8..a225e30f6c2 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,5 +1,3 @@ -import urllib - from spack import * class Openssl(Package): From 2b140b9a34852cbada6a3c1045b3b021d5eabeb8 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 1 Feb 2016 11:46:33 -0500 Subject: [PATCH 097/339] hdf5: default +unsupported to on This flag can end up being required if a dependency tree ends up needing hdf5+cxx+parallel, but nothing turns it on. Since the core spack code puts this together, it ends up that the end user needs to specify it manually. Instead, just assume an unsupported configuration (since these *are* useful setups) and if anyone wants to seek upstream support, masking the flag can be done. --- var/spack/repos/builtin/packages/hdf5/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 5321a191f0e..80f79539c09 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -45,7 +45,7 @@ class Hdf5(Package): variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') - variant('unsupported', default=False, description='Enables unsupported configuration options') + variant('unsupported', default=True, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') variant('szip', default=False, description='Enable szip support') From 422e87badbd073a9ca19e8c2e781a945d3a8e511 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 10:57:58 -0600 Subject: [PATCH 098/339] Add latest libgpg-error version --- var/spack/repos/builtin/packages/libgpg-error/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py index 6c1d1a10a7e..dd5fc2408a9 100644 --- a/var/spack/repos/builtin/packages/libgpg-error/package.py +++ b/var/spack/repos/builtin/packages/libgpg-error/package.py @@ -9,6 +9,7 @@ class LibgpgError(Package): homepage = "https://www.gnupg.org/related_software/libgpg-error" url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" + version('1.21', 'ab0b5aba6d0a185b41d07bda804fd8b2') version('1.18', '12312802d2065774b787cbfc22cc04e9') def install(self, spec, prefix): From ab3698e3a45e96b8cdd8d99220011350fa0489d1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 11:42:31 -0600 Subject: [PATCH 099/339] Add latest qhull version --- var/spack/repos/builtin/packages/qhull/package.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index f6712ced38c..bdca6db15dd 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -8,20 +8,18 @@ class Qhull(Package): implements the Quickhull algorithm for computing the convex hull. It handles roundoff errors from floating point arithmetic. It computes volumes, surface areas, and - approximations to the convex hull. - - Qhull does not support triangulation of non-convex surfaces, - mesh generation of non-convex objects, medium-sized inputs in - 9-D and higher, alpha shapes, weighted Voronoi diagrams, - Voronoi volumes, or constrained Delaunay triangulations.""" + approximations to the convex hull.""" homepage = "http://www.qhull.org" + version('7.2.0', 'e6270733a826a6a7c32b796e005ec3dc', + url="http://www.qhull.org/download/qhull-2015-src-7.2.0.tgz") + version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', url="http://www.qhull.org/download/qhull-2012.1-src.tgz") # https://github.com/qhull/qhull/pull/5 - patch('qhull-iterator.patch') + patch('qhull-iterator.patch', when='@1.0') def install(self, spec, prefix): with working_dir('spack-build', create=True): From d2fb1522a3524315a036cd5d46a99b2e05f1ad62 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 2 Feb 2016 12:54:24 -0600 Subject: [PATCH 100/339] Add Phonopy and PyYAML Python modules --- .../builtin/packages/py-phonopy/package.py | 18 ++++++++++++++++++ .../builtin/packages/py-pyyaml/package.py | 13 +++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 var/spack/repos/builtin/packages/py-phonopy/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyyaml/package.py diff --git a/var/spack/repos/builtin/packages/py-phonopy/package.py b/var/spack/repos/builtin/packages/py-phonopy/package.py new file mode 100644 index 00000000000..6d10fea74ff --- /dev/null +++ b/var/spack/repos/builtin/packages/py-phonopy/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyPhonopy(Package): + """Phonopy is an open source package for phonon + calculations at harmonic and quasi-harmonic levels.""" + homepage = "http://atztogo.github.io/phonopy/index.html" + url = "http://sourceforge.net/projects/phonopy/files/phonopy/phonopy-1.10/phonopy-1.10.0.tar.gz" + + version('1.10.0', '973ed1bcea46e21b9bf747aab9061ff6') + + extends('python') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-matplotlib') + depends_on('py-pyyaml') + + def install(self, spec, prefix): + python('setup.py', 'install', '--home=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyyaml/package.py b/var/spack/repos/builtin/packages/py-pyyaml/package.py new file mode 100644 index 00000000000..cae42f6e591 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyyaml/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyyaml(Package): + """PyYAML is a YAML parser and emitter for Python.""" + homepage = "http://pyyaml.org/wiki/PyYAML" + url = "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" + + version('3.11', 'f50e08ef0fe55178479d3a618efe21db') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 587d356d6ec8820cf04043b3650ac66b82a651b2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 3 Feb 2016 08:58:34 +0100 Subject: [PATCH 101/339] openssl : added logic to version computation. The package now warns user if he depends on an old version of the library --- .../repos/builtin/packages/openssl/package.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 05de35fca06..c73102f05d3 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,3 +1,6 @@ +import urllib +import llnl.util.tty as tty + from spack import * class Openssl(Package): @@ -10,6 +13,7 @@ class Openssl(Package): url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.1r', '1abd905e079542ccae948af37e393d28') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') @@ -17,6 +21,42 @@ class Openssl(Package): depends_on("zlib") parallel = False + def url_for_version(self, version): + # This URL is computed pinging the place where the latest version is stored. To avoid slowdown + # due to repeated pinging, we store the URL in a private class attribute to do the job only once per version + openssl_urls = getattr(Openssl, '_openssl_url', {}) + openssl_url = openssl_urls.get(version, None) + # Same idea, but just to avoid issuing the same message multiple times + warnings_given_to_user = getattr(Openssl, '_warnings_given', {}) + if openssl_url is None: + latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' + older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' + # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then + # return the url for older format + version_number = '.'.join([str(x) for x in version[:-1]]) + older_url = older.format(version_number=version_number, version_full=version) + latest_url = latest.format(version=version) + response = urllib.urlopen(latest.format(version=version)) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: + openssl_url = latest_url + # Store the computed URL + openssl_urls[version] = openssl_url + # Store the updated dictionary of URLS + Openssl._openssl_url = openssl_urls + # Store the updated dictionary of warnings + Openssl._warnings_given = warnings_given_to_user + + return openssl_url + def install(self, spec, prefix): # OpenSSL uses a variable APPS in its Makefile. If it happens to be set # in the environment, then this will override what is set in the From 44c1b06609925307e0e522b10daecf1dbe2eddc4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 1 Feb 2016 10:53:27 +0100 Subject: [PATCH 102/339] opencv : added package --- .../repos/builtin/packages/opencv/package.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 var/spack/repos/builtin/packages/opencv/package.py diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py new file mode 100644 index 00000000000..99b555323f6 --- /dev/null +++ b/var/spack/repos/builtin/packages/opencv/package.py @@ -0,0 +1,50 @@ +from spack import * + + +class Opencv(Package): + """ + OpenCV is released under a BSD license and hence it's free for both academic and commercial use. It has C++, C, + Python and Java interfaces and supports Windows, Linux, Mac OS, iOS and Android. OpenCV was designed for + computational efficiency and with a strong focus on real-time applications. Written in optimized C/C++, the library + can take advantage of multi-core processing. Enabled with OpenCL, it can take advantage of the hardware + acceleration of the underlying heterogeneous compute platform. Adopted all around the world, OpenCV has more than + 47 thousand people of user community and estimated number of downloads exceeding 9 million. Usage ranges from + interactive art, to mines inspection, stitching maps on the web or through advanced robotics. + """ + homepage = 'http://opencv.org/' + url = 'https://github.com/Itseez/opencv/archive/3.1.0.tar.gz' + + version('3.1.0', '70e1dd07f0aa06606f1bc0e3fa15abd3') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + variant('eigen', default=True, description='Activates support for eigen') + variant('ipp', default=True, description='Activates support for IPP') + + depends_on('zlib') + depends_on('libpng') + depends_on('libjpeg-turbo') + depends_on('libtiff') + + depends_on('python') + depends_on('py-numpy') + + depends_on('eigen', when='+eigen') + + # FIXME : GUI extensions missing + # FIXME : CUDA extensions still missing + + def install(self, spec, prefix): + cmake_options = [] + cmake_options.extend(std_cmake_args) + + cmake_options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DENABLE_PRECOMPILED_HEADERS:BOOL=OFF', + '-DWITH_IPP:BOOL=%s' % ('ON' if '+ipp' in spec else 'OFF')]) + + with working_dir('spack_build', create=True): + cmake('..', *cmake_options) + make('VERBOSE=1') + make("install") From b6434d9581b8beb2457504c008dca5a035cce182 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 12:09:43 -0800 Subject: [PATCH 103/339] Update mailmap --- .mailmap | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 1b99da32b5b..39ec1832413 100644 --- a/.mailmap +++ b/.mailmap @@ -1,13 +1,20 @@ Todd Gamblin George Todd Gamblin +Todd Gamblin Todd Gamblin Adam Moody Adam T. Moody Alfredo Gimenez Alfredo Gimenez David Boehme David Boehme David Boehme David Boehme Kevin Brandstatter Kevin Brandstatter Luc Jaulmes Luc Jaulmes -Saravan Pantham Saravan Pantham Saravan Pantham Tom Scogland Tom Scogland Tom Scogland Tom Scogland Joachim Protze jprotze Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory L. Lee Gregory L. Lee Gregory Lee +Massimiliano Culpo Massimiliano Culpo +Massimiliano Culpo alalazo +Mark Miller miller86 From e1b3c286ef3817c63b9566d6b64ea330fc739903 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 3 Feb 2016 16:28:09 -0600 Subject: [PATCH 104/339] Add url for cmake, simplify url_for_version --- var/spack/repos/builtin/packages/cmake/package.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 408a90460b9..e20c1e4aeb0 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -23,13 +23,14 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import llnl.util.tty as tty class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of tools designed to build, test and package software.""" homepage = 'https://www.cmake.org' + url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f') @@ -40,12 +41,7 @@ class Cmake(Package): def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" - parts = [str(p) for p in Version(version)] - if len(parts) < 3: - tty.error("Version '%s'does not match CMake's version naming scheme (z.y.x)." % version) - version_short = ".".join(parts[:2]) - version_full = ".".join(parts) - return "http://www.cmake.org/files/v%s/cmake-%s.tar.gz" % (version_short,version_full) + return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) def install(self, spec, prefix): From 031c292136d18889e5cd6e8facb68bf2c50ec083 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 11:19:25 +0100 Subject: [PATCH 105/339] dakota : basic installation --- .../repos/builtin/packages/dakota/package.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 var/spack/repos/builtin/packages/dakota/package.py diff --git a/var/spack/repos/builtin/packages/dakota/package.py b/var/spack/repos/builtin/packages/dakota/package.py new file mode 100644 index 00000000000..5ec82ef83d0 --- /dev/null +++ b/var/spack/repos/builtin/packages/dakota/package.py @@ -0,0 +1,55 @@ +from spack import * + + +class Dakota(Package): + """ + The Dakota toolkit provides a flexible, extensible interface between analysis codes and iterative systems + analysis methods. Dakota contains algorithms for: + + - optimization with gradient and non gradient-based methods; + - uncertainty quantification with sampling, reliability, stochastic expansion, and epistemic methods; + - parameter estimation with nonlinear least squares methods; + - sensitivity/variance analysis with design of experiments and parameter study methods. + + These capabilities may be used on their own or as components within advanced strategies such as hybrid optimization, + surrogate-based optimization, mixed integer nonlinear programming, or optimization under uncertainty. + """ + + homepage = 'https://dakota.sandia.gov/' + url = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-6.3-public.src.tar.gz' + _url_str = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-{version}-public.src.tar.gz' + + version('6.3', '05a58d209fae604af234c894c3f73f6d') + + variant('debug', default=False, description='Builds a debug version of the libraries') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + depends_on('python') + depends_on('boost') + + def url_for_version(self, version): + return Dakota._url_str.format(version=version) + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + + options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF')]) + + if '+mpi' in spec: + options.extend(['-DDAKOTA_HAVE_MPI:BOOL=ON', + '-DMPI_CXX_COMPILER:STRING=%s' % join_path(spec['mpi'].prefix.bin, 'mpicxx')]) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") From e3a0e1881dce3b251608d4252c7aafaa2da34104 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 16:00:39 +0100 Subject: [PATCH 106/339] suitesparse : basic implementation --- .../builtin/packages/SuiteSparse/package.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/repos/builtin/packages/SuiteSparse/package.py diff --git a/var/spack/repos/builtin/packages/SuiteSparse/package.py b/var/spack/repos/builtin/packages/SuiteSparse/package.py new file mode 100644 index 00000000000..6e130d118fe --- /dev/null +++ b/var/spack/repos/builtin/packages/SuiteSparse/package.py @@ -0,0 +1,27 @@ +from spack import * + + +class Suitesparse(Package): + """ + SuiteSparse is a suite of sparse matrix algorithms + """ + homepage = 'http://faculty.cse.tamu.edu/davis/suitesparse.html' + url = 'http://faculty.cse.tamu.edu/davis/SuiteSparse/SuiteSparse-4.5.1.tar.gz' + + version('4.5.1', 'f0ea9aad8d2d1ffec66a5b6bfeff5319') + + depends_on('blas') + depends_on('lapack') + + depends_on('metis@5.1.0', when='@4.5.1') + + def install(self, spec, prefix): + # The build system of SuiteSparse is quite old-fashioned + # It's basically a plain Makefile which include an header (SuiteSparse_config/SuiteSparse_config.mk) + # with a lot of convoluted logic in it. + # Any kind of customization will need to go through filtering of that file + + # FIXME : this actually uses the current workaround + # FIXME : (blas / lapack always provide libblas and liblapack as aliases) + make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') + From e2a95d4b5025a9965ab231e0c43fe9bebd88ca92 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 4 Feb 2016 16:27:26 +0100 Subject: [PATCH 107/339] eigen : added support for SuiteSparse --- var/spack/repos/builtin/packages/eigen/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 44ee6819f58..e40046b4529 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -41,13 +41,14 @@ class Eigen(Package): variant('metis', default=True, description='Enables metis backend') variant('scotch', default=True, description='Enables scotch backend') variant('fftw', default=True, description='Enables FFTW backend') + variant('suitesparse', default=True, description='Enables SuiteSparse support') - # TODO : dependency on SuiteSparse, googlehash, superlu, adolc missing + # TODO : dependency on googlehash, superlu, adolc missing depends_on('metis', when='+metis') depends_on('scotch', when='+scotch') depends_on('fftw', when='+fftw') - + depends_on('SuiteSparse', when='+suitesparse') depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 depends_on('gmp') From 71a25a109a8c9f23ce9aab0743ebf4516020519d Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 5 Feb 2016 11:00:37 +0100 Subject: [PATCH 108/339] arpack-ng : added package --- .../builtin/packages/arpack-ng/package.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 var/spack/repos/builtin/packages/arpack-ng/package.py diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py new file mode 100644 index 00000000000..0b49d142024 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -0,0 +1,57 @@ +from spack import * + + +class ArpackNg(Package): + """ + ARPACK-NG is a collection of Fortran77 subroutines designed to solve large scale eigenvalue problems. + + Important Features: + + * Reverse Communication Interface. + * Single and Double Precision Real Arithmetic Versions for Symmetric, + Non-symmetric, Standard or Generalized Problems. + * Single and Double Precision Complex Arithmetic Versions for Standard or + Generalized Problems. + * Routines for Banded Matrices - Standard or Generalized Problems. + * Routines for The Singular Value Decomposition. + * Example driver routines that may be used as templates to implement numerous + Shift-Invert strategies for all problem types, data types and precision. + + This project is a joint project between Debian, Octave and Scilab in order to + provide a common and maintained version of arpack. + + Indeed, no single release has been published by Rice university for the last + few years and since many software (Octave, Scilab, R, Matlab...) forked it and + implemented their own modifications, arpack-ng aims to tackle this by providing + a common repository and maintained versions. + + arpack-ng is replacing arpack almost everywhere. + """ + homepage = 'https://github.com/opencollab/arpack-ng' + url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' + + version('3.3.0', 'ed3648a23f0a868a43ef44c97a21bad5') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=False, description='Activates MPI support') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + # Apparently autotools are not bootstrapped + bootstrap = Executable('./bootstrap') + + options = ['--prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-mpi') + + if '~shared' in spec: + options.append('--enable-shared=no') + + bootstrap() + configure(*options) + make() + make('install') From f7134990bd9551d6361fb57db7095fa3bf44e092 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 9 Feb 2016 08:57:27 -0800 Subject: [PATCH 109/339] Fix #430: edit -f fails with UnknownPackageError - Recent external repo refactoring made `repo_for_pkg` raise an error when the package was not known. - Correct behavior is to return the highest precedence repo. --- lib/spack/spack/repository.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index b5df1168b66..f58cd52125b 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -300,8 +300,11 @@ def repo_for_pkg(self, spec): for repo in self.repos: if spec.name in repo: return repo - else: - raise UnknownPackageError(spec.name) + + # If the package isn't in any repo, return the one with + # highest precedence. This is for commands like `spack edit` + # that can operate on packages that don't exist yet. + return self.first_repo() @_autospec From 25e2be3efda2f70cf8d6c4e7b6089fbe9124c0d3 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 13:28:01 +0100 Subject: [PATCH 110/339] RPATH was collecting all lib from all dependencies, when dep tree is deep like in ParaView the path was growing too much, showing install problems now just direct dependencies rpath is added --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index b2db83acb7b..1b877780801 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -237,9 +237,9 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths From f729cf621ad463c953a81f4963b87535508583da Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 13:29:36 +0100 Subject: [PATCH 111/339] fixing qhull: seems that dependency on cmake is essential when it is used --- var/spack/repos/builtin/packages/qhull/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index bdca6db15dd..8f7c2f31b12 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -20,6 +20,8 @@ class Qhull(Package): # https://github.com/qhull/qhull/pull/5 patch('qhull-iterator.patch', when='@1.0') + + depends_on('cmake') def install(self, spec, prefix): with working_dir('spack-build', create=True): From fe8aca630c65e1f0324bfebeca11ca1cb3d4183c Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 14:58:00 +0100 Subject: [PATCH 112/339] add a variant to Qt for make dependency on mesa swlwctable --- var/spack/repos/builtin/packages/qt/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index e8d843519de..91afa420c17 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -23,6 +23,7 @@ class Qt(Package): version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009', url="http://download.qt.io/archive/qt/3/qt-x11-free-3.3.8b.tar.gz") + variant('mesa', default=False, description='depend on mesa') # Add patch for compile issues with qt3 found with use in the OpenSpeedShop project variant('krellpatch', default=False, description="build with openspeedshop based patch.") patch('qt3krell.patch', when='@3.3.8b+krellpatch') @@ -48,7 +49,7 @@ class Qt(Package): # depends_on("icu4c") # OpenGL hardware acceleration - depends_on("mesa", when='@4:') + depends_on("mesa", when='@4:+mesa') depends_on("libxcb") From a1bd65550e343c25ffcaf22aadf7a1875b08d853 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 15:17:54 +0100 Subject: [PATCH 113/339] fix a reported hash mismatch on netcdf-4.4.0 tar download --- var/spack/repos/builtin/packages/netcdf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 89b40f4a90d..41a0d2b6f92 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -9,7 +9,7 @@ class Netcdf(Package): homepage = "http://www.unidata.ucar.edu/software/netcdf/" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" - version('4.4.0', 'f01cb26a0126dd9a6224e76472d25f6c') + version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('mpi', default=True, description='Enables MPI parallelism') From d7b3ed08ab15caa3aac95700d84bfb52d8e1ac2f Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 10 Feb 2016 15:40:05 +0100 Subject: [PATCH 114/339] add variant to select OpenGL2 Paraview backend --- var/spack/repos/builtin/packages/paraview/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index aaab352e662..e43bdd44930 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -14,6 +14,7 @@ class Paraview(Package): variant('osmesa', default=False, description='Enable OSMesa support') variant('qt', default=False, description='Enable Qt support') + variant('opengl2', default=False, description='Enable OPengl2 backend') depends_on('python', when='+python') depends_on('py-numpy', when='+python') From 157ec210a74c2897abc093c14824975c49944f16 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 13:05:31 +0100 Subject: [PATCH 115/339] espresso : synchronized with current develop --- var/spack/{ => repos/builtin}/packages/espresso/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/{ => repos/builtin}/packages/espresso/package.py (100%) diff --git a/var/spack/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py similarity index 100% rename from var/spack/packages/espresso/package.py rename to var/spack/repos/builtin/packages/espresso/package.py From ca3cdb445825126776c5269481540d3afac02c9f Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 17:45:09 +0100 Subject: [PATCH 116/339] espresso : current working tree --- .../builtin/packages/espresso/package.py | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ce5dcc2accc..56b8c056b8a 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,53 +1,44 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install espresso -# -# You can always get back here to change things with: -# -# spack edit espresso -# -# See the spack documentation for more information on building -# packages. -# +import llnl.util.tty as tty + from spack import * + class Espresso(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://quantum-espresso.org" - url = "http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz" + """ + QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials + modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + """ + homepage = 'http://quantum-espresso.org' + url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') + variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') - variant('openmp', default=False, description='Build Quantum-ESPRESSO with mpi openmp') - variant('scalapack', default=False, description='Build Quantum-ESPRESSO with mpi openmp') + variant('openmp', default=False, description='Enables openMP support') + variant('scalapack', default=False, description='Enables scalapack support') + variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + depends_on('blas') + depends_on('lapack') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") depends_on('mpi', when='+mpi') + depends_on('elpa', when='+elpa') + depends_on('scalapack', when='+scalapack') - -# def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. -# configure('--prefix=%s' % prefix) - - # FIXME: Add logic to build and install here -# make() -# make("install") + def check_variants(self, spec): + error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' + if '+scalapack' in spec and '~mpi' in spec: + raise RuntimeError(error.format(variant='scalapack')) + if '+elpa' in spec and '~mpi' in spec: + raise RuntimeError(error.format(variant='elpa')) def install(self, spec, prefix): - # TAU isn't happy with directories that have '@' in the path. Sigh. + self.check_variants(spec) - # TAU configure, despite the name , seems to be a manually written script (nothing related to autotools). - # As such it has a few #peculiarities# that make this build quite hackish. - options = ["-prefix=%s" % prefix, - "--enable-parallel"] + options = ['-prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-parallel') if '+openmp' in spec: options.append('--enable-openmp') @@ -55,7 +46,9 @@ def install(self, spec, prefix): if '+scalapack' in spec: options.append('--with-scalapack=yes') - configure(*options) - make("all") - make("install") + if '+elpa' in spec: + options.append('--with-elpa=%s' % spec['elpa'].prefix) + configure(*options) + make('all') + make('install') From b272a8881becf205c09ac11228dd035b99fa0fcd Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 18:18:05 +0100 Subject: [PATCH 117/339] espresso : fixed dependency handling --- var/spack/repos/builtin/packages/espresso/package.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 56b8c056b8a..df37bb0d71d 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,5 +1,3 @@ -import llnl.util.tty as tty - from spack import * @@ -15,21 +13,21 @@ class Espresso(Package): variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') variant('openmp', default=False, description='Enables openMP support') - variant('scalapack', default=False, description='Enables scalapack support') + variant('scalapack', default=True, description='Enables scalapack support') variant('elpa', default=True, description='Use elpa as an eigenvalue solver') depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') - depends_on('elpa', when='+elpa') - depends_on('scalapack', when='+scalapack') + depends_on('elpa', when='+elpa+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: raise RuntimeError(error.format(variant='scalapack')) - if '+elpa' in spec and '~mpi' in spec: + if '+elpa' in spec and ('~mpi' in spec or '~scalapack' in spec): raise RuntimeError(error.format(variant='elpa')) def install(self, spec, prefix): From e8704433debe35d7893c0c20672d06973226c4e8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 11 Feb 2016 18:57:40 +0100 Subject: [PATCH 118/339] espresso : added directories to search path --- var/spack/repos/builtin/packages/espresso/package.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index df37bb0d71d..f85257b3ccf 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -47,6 +47,14 @@ def install(self, spec, prefix): if '+elpa' in spec: options.append('--with-elpa=%s' % spec['elpa'].prefix) + # Add a list of directories to search + search_list = [] + for name, dependency_spec in spec.dependencies.iteritems(): + print name + search_list.extend([dependency_spec.prefix.lib, + dependency_spec.prefix.lib64]) + search_list = " ".join(search_list) + options.append('LIBDIRS=%s' % search_list) configure(*options) make('all') make('install') From d7f674ce9678ddb3b964ad457b29ed058e2cb3c7 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 14 Jan 2016 17:41:44 -0500 Subject: [PATCH 119/339] libedit depends on ncurses --- var/spack/repos/builtin/packages/libedit/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/libedit/package.py b/var/spack/repos/builtin/packages/libedit/package.py index bcd5212b9ea..faed8bad374 100644 --- a/var/spack/repos/builtin/packages/libedit/package.py +++ b/var/spack/repos/builtin/packages/libedit/package.py @@ -7,6 +7,8 @@ class Libedit(Package): version('3.1', '43cdb5df3061d78b5e9d59109871b4f6', url="http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz") + depends_on('ncurses') + def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 70985170e5f248d0d6a1b6245bd5581185ecaa64 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 12 Feb 2016 12:08:31 +0100 Subject: [PATCH 120/339] qe : compiled on rhle6 --- .../repos/builtin/packages/espresso/package.py | 15 ++++++++++----- var/spack/repos/builtin/packages/mpich/package.py | 8 ++++---- .../repos/builtin/packages/openblas/package.py | 2 ++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index f85257b3ccf..a2bf58f585b 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -1,5 +1,6 @@ from spack import * +import os class Espresso(Package): """ @@ -20,9 +21,10 @@ class Espresso(Package): depends_on('lapack') depends_on('mpi', when='+mpi') - depends_on('elpa', when='+elpa+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + depends_on('fftw~mpi', when='~mpi') + depends_on('fftw+mpi', when='+mpi') depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation - + def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: @@ -45,16 +47,19 @@ def install(self, spec, prefix): options.append('--with-scalapack=yes') if '+elpa' in spec: - options.append('--with-elpa=%s' % spec['elpa'].prefix) + options.append('--with-elpa=yes') # Add a list of directories to search search_list = [] for name, dependency_spec in spec.dependencies.iteritems(): - print name search_list.extend([dependency_spec.prefix.lib, dependency_spec.prefix.lib64]) + search_list = " ".join(search_list) options.append('LIBDIRS=%s' % search_list) + options.append('F90=%s' % os.environ['FC']) + configure(*options) make('all') - make('install') + make('install') + diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c856cfe277f..26d3bc0c945 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -48,10 +48,10 @@ class Mpich(Package): def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = 'cc' - os.environ['MPICH_CXX'] = 'c++' - os.environ['MPICH_F77'] = 'f77' - os.environ['MPICH_F90'] = 'f90' + os.environ['MPICH_CC'] = os.environ['CC'] + os.environ['MPICH_CXX'] = os.environ['CXX'] + os.environ['MPICH_F77'] = os.environ['F77'] + os.environ['MPICH_FC'] = os.environ['FC'] def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 9c8fa1c694b..3c909360a41 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -19,7 +19,9 @@ def install(self, spec, prefix): with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') + symlink('libopenblas.so', 'libblas.so') # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') + symlink('libopenblas.so', 'liblapack.so') From 247a4bc75108aa2106c1b191345a0c3387a8afac Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 12 Feb 2016 12:15:09 +0100 Subject: [PATCH 121/339] mpich : added back MPI_F90 --- var/spack/repos/builtin/packages/mpich/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 26d3bc0c945..c517defa83e 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -51,6 +51,7 @@ def setup_dependent_environment(self, module, spec, dep_spec): os.environ['MPICH_CC'] = os.environ['CC'] os.environ['MPICH_CXX'] = os.environ['CXX'] os.environ['MPICH_F77'] = os.environ['F77'] + os.environ['MPICH_F90'] = os.environ['FC'] os.environ['MPICH_FC'] = os.environ['FC'] From 90e1b1f0ea4320a5bdc0f54ec14a12fdda6584ff Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 12 Feb 2016 09:33:19 -0800 Subject: [PATCH 122/339] Use hash to make shorter module and dotkit names, fixes #433 --- lib/spack/spack/modules.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7036626e29e..c8347635646 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -194,12 +194,14 @@ class Dotkit(EnvModule): @property def file_name(self): return join_path(Dotkit.path, self.spec.architecture, - self.spec.format('$_$@$%@$+$#.dk')) + '%s.dk' % self.use_name) @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') - + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) def _write(self, dk_file): # Category @@ -235,7 +237,10 @@ def file_name(self): @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) def _write(self, m_file): From 2cd9ad8ce63ea29093fcc2d8e1bd749d5cbccf0b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 17 Jan 2016 10:46:21 -0500 Subject: [PATCH 123/339] Use "-Wl,-rpath," instead of "-Wl,-rpath=" The former translates to a linker argument "-rpath DIR", whereas the latter translates to "-rpath=DIR". The latter is not support on OS X. --- lib/spack/docs/packaging_guide.rst | 6 +++--- lib/spack/spack/package.py | 4 ++-- lib/spack/spack/test/cc.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bb8a26ad02f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1711,15 +1711,15 @@ Compile-time library search paths * ``-L$dep_prefix/lib`` * ``-L$dep_prefix/lib64`` Runtime library search paths (RPATHs) - * ``-Wl,-rpath=$dep_prefix/lib`` - * ``-Wl,-rpath=$dep_prefix/lib64`` + * ``-Wl,-rpath,$dep_prefix/lib`` + * ``-Wl,-rpath,$dep_prefix/lib64`` Include search paths * ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, -``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib`` +``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` inserted on the command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8cb947c276c..49d4fb6b237 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1220,8 +1220,8 @@ def rpath(self): @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath= for each element.""" - return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) + """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) def validate_package_url(url_string): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 905af28a06d..54d56383941 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -39,11 +39,11 @@ 'arg1', '-Wl,--start-group', 'arg2', - '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', + '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] @@ -95,13 +95,13 @@ def test_cpp_mode(self): def test_ccld_mode(self): self.check_cc('dump-mode', [], "ccld") self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld") - self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") - self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") + self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") + self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") def test_ld_mode(self): self.check_ld('dump-mode', [], "ld") - self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld") + self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") def test_includes(self): From 5038a38e296940f463c13d6fdda728d8673bc95a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 18 Jan 2016 18:13:18 -0500 Subject: [PATCH 124/339] Correct -Xlinker arguments --- lib/spack/spack/test/cc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 54d56383941..11420ec44a1 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath,/fourth/rpath', + '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From dc6a33b716dd6712da2e65a78a6a3ed98ca72d4d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:09:29 -0500 Subject: [PATCH 125/339] Handle multiple -Wl,-rpath,... paths --- lib/spack/env/cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..c156b7b6073 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,14 +176,21 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + if [[ $arg = -rpath=* ]]; then + arg="${arg#-rpath=}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then + if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - rpaths+=("${arg#-Wl,}") + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + arg="${arg#-Wl,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done else other_args+=("-Wl,$arg") fi @@ -191,11 +198,11 @@ while [ -n "$1" ]; do -Xlinker,*) arg="${1#-Xlinker,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then + if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then + if [[ $arg != -Xlinker,* ]]; then die "-Xlinker,-rpath was not followed by -Xlinker,*" fi rpaths+=("${arg#-Xlinker,}") From 9a2c1090a6f40db762ac155d7d25063c2965c841 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Fri, 12 Feb 2016 12:37:03 -0500 Subject: [PATCH 126/339] Handle -Wl,-rpath,... syntax --- lib/spack/env/cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c156b7b6073..41933f5e1fe 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -181,6 +181,11 @@ while [ -n "$1" ]; do for rpath in ${arg//,/ }; do rpaths+=("$rpath") done + elif [[ $arg = -rpath,* ]]; then + arg="${arg#-rpath,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done elif [[ $arg = -rpath ]]; then shift; arg="$1" if [[ $arg != -Wl,* ]]; then From 69064395eb61db3b03d1ed14f16bef7ec2c94ee3 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 15:12:09 -0500 Subject: [PATCH 127/339] Add debug output --- lib/spack/env/cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 41933f5e1fe..a431cffacf4 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,19 +174,26 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) + echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi + echo "SHIFTED, arg=[$arg]" >&2 if [[ $arg = -rpath=* ]]; then + echo "CASE 1" >&2 arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath,* ]]; then + echo "CASE 2" >&2 arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then + echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -194,9 +201,11 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do + echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else + echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 52647b9a5d8ab4b2fff7387f65347164576be088 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 17:55:14 -0500 Subject: [PATCH 128/339] Using regexes instead of globbing to match path names --- lib/spack/env/cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a431cffacf4..c46986e19cd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,26 +174,19 @@ while [ -n "$1" ]; do libs+=("$arg") ;; -Wl,*) - echo "FOUND arg=[$arg]" >&2 arg="${1#-Wl,}" if [ -z "$arg" ]; then shift; arg="$1"; fi - echo "SHIFTED, arg=[$arg]" >&2 - if [[ $arg = -rpath=* ]]; then - echo "CASE 1" >&2 + if [[ $arg =~ -rpath=.* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done - elif [[ $arg = -rpath,* ]]; then - echo "CASE 2" >&2 + elif [[ $arg =~ -rpath,.* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 - rpaths+=("$rpath") + rpaths+=("$rpath") done elif [[ $arg = -rpath ]]; then - echo "CASE 3" >&2 shift; arg="$1" if [[ $arg != -Wl,* ]]; then die "-Wl,-rpath was not followed by -Wl,*" @@ -201,11 +194,9 @@ while [ -n "$1" ]; do # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do - echo " RPATH=[$rpath]" >&2 rpaths+=("$rpath") done else - echo "OTHER" >&2 other_args+=("-Wl,$arg") fi ;; From 43670cbbd01562f6748f3ee6e2505be52e24bccb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 13 Feb 2016 22:01:36 -0500 Subject: [PATCH 129/339] More games with quoting --- lib/spack/env/cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c46986e19cd..37483e00733 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,20 +175,19 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ $arg =~ -rpath=.* ]]; then + if [[ $arg = "-rpath=*" ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg =~ -rpath,.* ]]; then + elif [[ $arg = "-rpath,*" ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do - rpaths+=("$rpath") + rpaths+=("$rpath") done - elif [[ $arg = -rpath ]]; then + elif [[ $arg = "-rpath" ]]; then shift; arg="$1" - if [[ $arg != -Wl,* ]]; then + if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi # TODO: Handle multiple -Wl, continuations of -Wl,-rpath From a06e29fecbaf100f089c1eab7c6447cd469b95f6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 10:43:55 -0500 Subject: [PATCH 130/339] More quoting experiments --- lib/spack/env/cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 37483e00733..853a19dfddd 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -175,22 +175,22 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [[ $arg = "-rpath=*" ]]; then + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + if [[ $arg == '-rpath='* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath,*" ]]; then + elif [[ $arg == '-rpath,'* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg = "-rpath" ]]; then + elif [[ $arg == '-rpath' ]]; then shift; arg="$1" if [[ $arg != "-Wl,*" ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - # TODO: Handle multiple -Wl, continuations of -Wl,-rpath arg="${arg#-Wl,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") From 9868333e8e2586ffc55ee996a48ef62601a9c874 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 14 Feb 2016 11:34:08 -0500 Subject: [PATCH 131/339] Shell quoting is difficult --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 853a19dfddd..c3d11357224 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -188,7 +188,7 @@ while [ -n "$1" ]; do done elif [[ $arg == '-rpath' ]]; then shift; arg="$1" - if [[ $arg != "-Wl,*" ]]; then + if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi arg="${arg#-Wl,}" From d8c0edcc818b8f0fb2220993d469381ae072ebe4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 14 Feb 2016 14:40:25 -0800 Subject: [PATCH 132/339] Minor code cleanup for gcc. --- var/spack/repos/builtin/packages/gcc/package.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 3e5895cfb83..f8958ee2907 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -36,8 +36,6 @@ class Gcc(Package): list_url = 'http://open-source-box.org/gcc/' list_depth = 2 - DEPENDS_ON_ISL_PREDICATE = '@5.0:' - version('5.3.0', 'c9616fd448f980259c31de613e575719') version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') version('4.9.3', '6f831b4d251872736e8e9cc09746f327') @@ -53,12 +51,11 @@ class Gcc(Package): depends_on("mpfr") depends_on("gmp") - depends_on("mpc") # when @4.5: + depends_on("mpc", when='@4.5:') + depends_on("isl", when='@5.0:') depends_on("binutils~libiberty", when='~gold') depends_on("binutils~libiberty+gold", when='+gold') - # Save these until we can do optional deps. - depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") @@ -91,7 +88,7 @@ def install(self, spec, prefix): "--with-as=%s/bin/as" % spec['binutils'].prefix] options.extend(binutils_options) # Isl - if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): + if 'isl' in spec: isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) From bf162e60f17f8aebcd62184d72a194eb91e4ef4f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 15 Feb 2016 10:53:50 -0600 Subject: [PATCH 133/339] Add latest version --- var/spack/repos/builtin/packages/py-mpi4py/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py index 8001689a18d..f599205644e 100644 --- a/var/spack/repos/builtin/packages/py-mpi4py/package.py +++ b/var/spack/repos/builtin/packages/py-mpi4py/package.py @@ -5,7 +5,9 @@ class PyMpi4py(Package): homepage = "https://pypi.python.org/pypi/mpi4py" url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" + version('2.0.0', '4f7d8126d7367c239fd67615680990e3') version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') + extends('python') depends_on('py-setuptools') depends_on('mpi') From db50f52bbc47ef6dbe2e26c54ce828c0e6076e96 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 11:40:00 -0800 Subject: [PATCH 134/339] Adding the "Crypto++" package installation files. --- .../builtin/packages/cryptopp/package.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 var/spack/repos/builtin/packages/cryptopp/package.py diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py new file mode 100644 index 00000000000..1693c4b160f --- /dev/null +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -0,0 +1,31 @@ +import glob +from spack import * + +class Cryptopp(Package): + """Crypto++ is an open-source C++ library of cryptographic schemes. The + library supports a number of different cryptography algorithms, including + authenticated encryption schemes (GCM, CCM), hash functions (SHA-1, SHA2), + public-key encryption (RSA, DSA), and a few obsolete/historical encryption + algorithms (MD5, Panama).""" + + homepage = "http://www.cryptopp.com/" + url = "http://www.cryptopp.com/cryptopp563.zip" + + version('5.6.3', '3c5b70e2ec98b7a24988734446242d07') + version('5.6.2', '7ed022585698df48e65ce9218f6c6a67') + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + for hfile in glob.glob('*.h*'): + install(hfile, prefix.include) + + mkdirp(prefix.lib) + install('libcryptopp.a', prefix.lib) + + def url_for_version(self, version): + version_tuple = tuple(v for v in iter(version)) + version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "") + + return "%scryptopp%s.zip" % (Cryptopp.homepage, version_string) From 9a6221ea40e8cda3bdfa495b7c4a93284e8f4895 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 12:47:14 -0800 Subject: [PATCH 135/339] Added the installation files for the "ndiff" package. --- .../repos/builtin/packages/ndiff/package.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 var/spack/repos/builtin/packages/ndiff/package.py diff --git a/var/spack/repos/builtin/packages/ndiff/package.py b/var/spack/repos/builtin/packages/ndiff/package.py new file mode 100644 index 00000000000..10e445c81e9 --- /dev/null +++ b/var/spack/repos/builtin/packages/ndiff/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Ndiff(Package): + """The ndiff tool is a binary utility that compares putatively similar files + while ignoring small numeric differernces. This utility is most often used + to compare files containing a lot of floating-point numeric data that + may be slightly different due to numeric error.""" + + homepage = "http://ftp.math.utah.edu/pub/ndiff/" + url = "http://ftp.math.utah.edu/pub/ndiff/ndiff-2.00.tar.gz" + + version('2.00', '885548b4dc26e72c5455bebb5ba6c16d') + version('1.00', 'f41ffe5d12f36cd36b6311acf46eccdc') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + mkdirp(prefix.bin) + mkdirp('%s/lib' % prefix.share) + + make('install-exe', 'install-shrlib') From 3c8bbeafc78c00cd93fa4526a0e55bf16d36b454 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 13:01:18 -0800 Subject: [PATCH 136/339] Added the installation files for the "Triangle" package. --- .../builtin/packages/Triangle/package.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 var/spack/repos/builtin/packages/Triangle/package.py diff --git a/var/spack/repos/builtin/packages/Triangle/package.py b/var/spack/repos/builtin/packages/Triangle/package.py new file mode 100644 index 00000000000..f65d93776df --- /dev/null +++ b/var/spack/repos/builtin/packages/Triangle/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Triangle(Package): + """Triangle is a two-dimensional mesh generator and Delaunay + triangulator. Triangle generates exact Delaunay triangulations, + constrained Delaunay triangulations, conforming Delaunay + triangulations, Voronoi diagrams, and high-quality triangular + meshes.""" + + homepage = "http://www.cs.cmu.edu/~quake/triangle.html" + url = "http://www.netlib.org/voronoi/triangle.zip" + + version('1.6', '10aff8d7950f5e0e2fb6dd2e340be2c9') + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + + install('triangle', prefix.bin) + install('showme', prefix.bin) From 09254014b182ccf4cbc4ce291141b7ab39b9171d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 13:24:01 -0800 Subject: [PATCH 137/339] Added the installation files for the "TetGen" package. --- .../repos/builtin/packages/tetgen/package.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/repos/builtin/packages/tetgen/package.py diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py new file mode 100644 index 00000000000..30c2b766558 --- /dev/null +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Tetgen(Package): + """TetGen is a program and library that can be used to generate tetrahedral + meshes for given 3D polyhedral domains. TetGen generates exact constrained + Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and + Voronoi paritions.""" + + homepage = "http://www.tetgen.org" + url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" + + version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:', when='@1.5.0:') + + def install(self, spec, prefix): + make('tetgen', 'tetlib') + + mkdirp(prefix.bin) + install('tetgen', prefix.bin) + + mkdirp(prefix.include) + install('tetgen.h', prefix.include) + + mkdirp(prefix.lib) + install('libtet.a', prefix.lib) From 8cab10214e63344b77ffc34df3609dbc9f1690a5 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 16:43:24 -0800 Subject: [PATCH 138/339] Added the installation files for the "ExodusII" package. Added the "+static" variant to "hdf5" to enable "ExodusII" support. --- .../packages/exodusii/exodus-cmake.patch | 12 +++++ .../builtin/packages/exodusii/package.py | 49 +++++++++++++++++++ .../repos/builtin/packages/hdf5/package.py | 4 ++ 3 files changed, 65 insertions(+) create mode 100644 var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch create mode 100644 var/spack/repos/builtin/packages/exodusii/package.py diff --git a/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch new file mode 100644 index 00000000000..25355269cae --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch @@ -0,0 +1,12 @@ +diff --git a/cmake-exodus b/cmake-exodus +index 787fd9d..ed073a2 100755 +--- a/cmake-exodus ++++ b/cmake-exodus +@@ -1,4 +1,6 @@ +-EXTRA_ARGS=$@ ++#!/bin/bash ++ ++EXTRA_ARGS=-DSEACASProj_ENABLE_CXX11=OFF + + ### Change this to point to the compilers you want to use + CC=gcc diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py new file mode 100644 index 00000000000..89c04bf79cf --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -0,0 +1,49 @@ +from spack import * + +# TODO: Add support for a C++11 enabled installation that filters out the +# TODO: "C++11-Disabled" flag (but only if the spec compiler supports C++11). + +# TODO: Add support for parallel installation that uses MPI. + +# TODO: Create installation options for NetCDF that support larger page size +# TODO: suggested by Exodus (see the repository "README" file). + +class Exodusii(Package): + """Exodus II is a C++/Fortran library developed to store and retrieve data for + finite element analyses. It's used for preprocessing (problem definition), + postprocessing (results visualization), and data transfer between codes. + An Exodus II data file is a random access, machine independent, binary + file that is written and read via C, C++, or Fortran API routines.""" + + homepage = "https://github.com/gsjaardema/seacas" + url = "https://github.com/gsjaardema/seacas/archive/master.zip" + + version('2016-02-08', git='https://github.com/gsjaardema/seacas.git', commit='dcf3529') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:') + depends_on('hdf5+static~mpi') + depends_on('netcdf~mpi') + + patch('exodus-cmake.patch') + + def patch(self): + ff = FileFilter('cmake-exodus') + + ff.filter('CMAKE_INSTALL_PREFIX:PATH=${ACCESS}', + 'CMAKE_INSTALL_PREFIX:PATH=%s' % self.spec.prefix, string=True) + ff.filter('NetCDF_DIR:PATH=${TPL}', + 'NetCDF_DIR:PATH=%s' % self.spec['netcdf'].prefix, string=True) + ff.filter('HDF5_ROOT:PATH=${TPL}', + 'HDF5_ROOT:PATH=%s' % self.spec['hdf5'].prefix, string=True) + + def install(self, spec, prefix): + mkdirp('build') + cd('build') + + cmake_exodus = Executable('../cmake-exodus') + cmake_exodus() + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 80f79539c09..7db4aff6315 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,6 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') + variant('static', default=False, description='Builds a static executable version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') @@ -78,6 +79,9 @@ def install(self, spec, prefix): else: extra_args.append('--enable-production') + if '+static' in spec: + extra_args.append('--enable-static-exec') + if '+unsupported' in spec: extra_args.append("--enable-unsupported") From fca7ef2f7b3d8091e935073ec24570ebb163fe6d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 17:23:37 -0800 Subject: [PATCH 139/339] Moved "Triangle" to "triangle" to be more in line with Spack package naming conventions. --- .../repos/builtin/packages/{Triangle => triangle}/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/repos/builtin/packages/{Triangle => triangle}/package.py (100%) diff --git a/var/spack/repos/builtin/packages/Triangle/package.py b/var/spack/repos/builtin/packages/triangle/package.py similarity index 100% rename from var/spack/repos/builtin/packages/Triangle/package.py rename to var/spack/repos/builtin/packages/triangle/package.py From 0388093f7aed92bd1d3282fe4f53c27e66030fd4 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 16 Feb 2016 17:33:19 -0800 Subject: [PATCH 140/339] Changed the hdf5 "+static" variant to become the "+shared" variant. --- var/spack/repos/builtin/packages/exodusii/package.py | 2 +- var/spack/repos/builtin/packages/hdf5/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py index 89c04bf79cf..d68baaa6d9d 100644 --- a/var/spack/repos/builtin/packages/exodusii/package.py +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -23,7 +23,7 @@ class Exodusii(Package): # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). depends_on('cmake@2.8.7:') - depends_on('hdf5+static~mpi') + depends_on('hdf5+shared~mpi') depends_on('netcdf~mpi') patch('exodus-cmake.patch') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 7db4aff6315..f4de92aa837 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,7 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') - variant('static', default=False, description='Builds a static executable version of the library') + variant('shared', default=False, description='Builds a static executable version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') From 6cd76d69c3fa4b9eb2d30660de28206697af722d Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Wed, 17 Feb 2016 11:51:24 -0800 Subject: [PATCH 141/339] Fixed the "+shared" variant in the "hdf5" package. --- var/spack/repos/builtin/packages/exodusii/package.py | 2 +- var/spack/repos/builtin/packages/hdf5/package.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py index d68baaa6d9d..af258b7e6e9 100644 --- a/var/spack/repos/builtin/packages/exodusii/package.py +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -23,7 +23,7 @@ class Exodusii(Package): # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). depends_on('cmake@2.8.7:') - depends_on('hdf5+shared~mpi') + depends_on('hdf5~shared~mpi') depends_on('netcdf~mpi') patch('exodus-cmake.patch') diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index f4de92aa837..ed4e7c35c99 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -42,7 +42,7 @@ class Hdf5(Package): version('1.8.13', 'c03426e9e77d7766944654280b467289') variant('debug', default=False, description='Builds a debug version of the library') - variant('shared', default=False, description='Builds a static executable version of the library') + variant('shared', default=True, description='Builds a shared version of the library') variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') @@ -79,7 +79,9 @@ def install(self, spec, prefix): else: extra_args.append('--enable-production') - if '+static' in spec: + if '+shared' in spec: + extra_args.append('--enable-shared') + else: extra_args.append('--enable-static-exec') if '+unsupported' in spec: @@ -123,7 +125,6 @@ def install(self, spec, prefix): configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-shared", # TODO : this should be enabled by default, remove it? *extra_args) make() make("install") From 00125e4688bb457937b4907f8e69afe2a06737fc Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 17 Feb 2016 16:43:36 -0800 Subject: [PATCH 142/339] remove the unprotected key name from create.py Small fix to use dicts correctly. --- lib/spack/spack/cmd/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index edcea0718cd..6809209046e 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -222,7 +222,7 @@ def fetch_tarballs(url, name, args): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { version : url } + versions = { "version" : url } elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( From d8a70166d3abd14b5a34025f735cf1825179f84f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 18 Feb 2016 00:56:29 -0800 Subject: [PATCH 143/339] Fixes #434 Compiler detection was not getting triggered properly with some of the new config logic. Adjust the conditions under which Spack will serach for compilers. --- lib/spack/spack/compilers/__init__.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6159ef576ca..3a04bc2ebc5 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -74,28 +74,36 @@ def _to_dict(compiler): def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ - # If any configuration file has compilers, just stick with the - # ones already configured. - config = spack.config.get_config('compilers', scope=scope) - + # Check whether we're on a front-end (native) architecture. my_arch = spack.architecture.sys_type() if arch is None: arch = my_arch - if arch in config: - return config[arch] - - # Only for the current arch in *highest* scope: automatically try to - # find compilers if none are configured yet. - if arch == my_arch and scope == 'user': + def init_compiler_config(): + """Compiler search used when Spack has no compilers.""" config[arch] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) - return config[arch] - return {} + config = spack.config.get_config('compilers', scope=scope) + + # Update the configuration if there are currently no compilers + # configured. Avoid updating automatically if there ARE site + # compilers configured but no user ones. + if arch == my_arch and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + + return config[arch] if arch in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): From 19d10291bfeae45315fcef852baddec63b69247a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 18 Feb 2016 15:45:29 -0800 Subject: [PATCH 144/339] modify compiler commands in python config files, fix for #431 --- var/spack/repos/builtin/packages/python/package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index a1ce06feb03..58d401244e3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -55,6 +55,20 @@ def install(self, spec, prefix): make() make("install") + # Modify compiler paths in configuration files. This is necessary for + # building site packages outside of spack + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + + python3_version = '' + if spec.satisfies('@3:'): + python3_version = '-%d.%dm' % self.version[:2] + makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + makefile_filepath) + # ======================================================================== # Set up environment to make install easy for python extensions. From 30c304748213bee3669c75d4384bad146f68dbd7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 19 Feb 2016 12:39:38 +0100 Subject: [PATCH 145/339] gromacs : added package --- .../repos/builtin/packages/fftw/package.py | 2 + .../repos/builtin/packages/gromacs/package.py | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 var/spack/repos/builtin/packages/gromacs/package.py diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 4d2b9642426..bc129aaf1a1 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -47,6 +47,8 @@ class Fftw(Package): depends_on('mpi', when='+mpi') + # TODO : add support for architecture specific optimizations as soon as targets are supported + def install(self, spec, prefix): options = ['--prefix=%s' % prefix, '--enable-shared', diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py new file mode 100644 index 00000000000..5fe8399308b --- /dev/null +++ b/var/spack/repos/builtin/packages/gromacs/package.py @@ -0,0 +1,56 @@ +from spack import * + + +class Gromacs(Package): + """ + GROMACS (GROningen MAchine for Chemical Simulations) is a molecular dynamics package primarily designed for + simulations of proteins, lipids and nucleic acids. It was originally developed in the Biophysical Chemistry + department of University of Groningen, and is now maintained by contributors in universities and research centers + across the world. + + GROMACS is one of the fastest and most popular software packages available and can run on CPUs as well as GPUs. + It is free, open source released under the GNU General Public License. Starting from version 4.6, GROMACS is + released under the GNU Lesser General Public License. + """ + + homepage = 'http://www.gromacs.org' + url = 'ftp://ftp.gromacs.org/pub/gromacs/gromacs-5.1.2.tar.gz' + + version('5.1.2', '614d0be372f1a6f1f36382b7a6fcab98') + + variant('mpi', default=True, description='Activate MPI support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Enables debug mode') + variant('double', default=False, description='Produces a double precision version of the executables') + + depends_on('mpi', when='+mpi') + + depends_on('fftw') + + # TODO : add GPU support + + def install(self, spec, prefix): + + options = [] + + if '+mpi' in spec: + options.append('-DGMX_MPI:BOOL=ON') + + if '+double' in spec: + options.append('-DGMX_DOUBLE:BOOL=ON') + + if '~shared' in spec: + options.append('-DBUILD_SHARED_LIBS:BOOL=OFF') + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + else: + options.append('-DCMAKE_BUILD_TYPE:STRING=Release') + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + + cmake('..', *options) + make() + make('install') From 5c8dd6c3c86400ac82b061ce45f58b543526bfff Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 20 Feb 2016 17:18:49 -0800 Subject: [PATCH 146/339] llvm trunk version Adding a trunk version to the llvm package. This has all the features and requirements of the others, with the additional caveat that the llvm project makes no guarantee that trunk on all repositories together will necessarily make a working compiler. It has been tested, and worked with a version today, but not yesterday, so if you test keep that in mind. --- .../repos/builtin/packages/llvm/package.py | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 1805d3ded80..a8f19f90711 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -117,6 +117,21 @@ class Llvm(Package): }, } releases = [ + { + 'version' : 'trunk', + 'repo' : 'http://llvm.org/svn/llvm-project/llvm/trunk', + 'resources' : { + 'compiler-rt' : 'http://llvm.org/svn/llvm-project/compiler-rt/trunk', + 'openmp' : 'http://llvm.org/svn/llvm-project/openmp/trunk', + 'polly' : 'http://llvm.org/svn/llvm-project/polly/trunk', + 'libcxx' : 'http://llvm.org/svn/llvm-project/libcxx/trunk', + 'libcxxabi' : 'http://llvm.org/svn/llvm-project/libcxxabi/trunk', + 'clang' : 'http://llvm.org/svn/llvm-project/cfe/trunk', + 'clang-tools-extra' : 'http://llvm.org/svn/llvm-project/clang-tools-extra/trunk', + 'lldb' : 'http://llvm.org/svn/llvm-project/lldb/trunk', + 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', @@ -161,15 +176,25 @@ class Llvm(Package): ] for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) + if release['version'] == 'trunk' : + version(release['version'], svn=release['repo']) - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + for name, repo in release['resources'].items(): + resource(name=name, + svn=repo, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) + else: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) # SVN - current develop version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') From 4316f1cd31afd18e35b5ebd9852a156d2c68c8df Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 23 Feb 2016 16:57:19 +0100 Subject: [PATCH 147/339] Modified wrapper to have a different behavior when modeis vcheck --- lib/spack/env/cc | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..dfb0801c569 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -113,14 +113,22 @@ case "$command" in ;; esac -# Finish setting up the mode. +# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries if [ -z "$mode" ]; then - mode=ccld for arg in "$@"; do if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then mode=vcheck break - elif [ "$arg" = -E ]; then + fi + done +fi + +# Finish setting up the mode. + +if [ -z "$mode" ]; then + mode=ccld + for arg in "$@"; do + if [ "$arg" = -E ]; then mode=cpp break elif [ "$arg" = -c ]; then @@ -145,6 +153,11 @@ fi # Save original command for debug logging input_command="$@" +if [ "$mode" == vcheck ] ; then + exec "${input_command}" + exit +fi + # # Now do real parsing of the command line args, trying hard to keep # non-rpath linker arguments in the proper order w.r.t. other command @@ -330,3 +343,5 @@ if [ "$SPACK_DEBUG" = "TRUE" ]; then fi exec "${full_command[@]}" + + From 5ce97eeda52944128dc32c0d7728550d95520b0d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 09:17:04 +0100 Subject: [PATCH 148/339] intel : polished code --- lib/spack/env/cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index dfb0801c569..644d2be1d66 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -155,7 +155,6 @@ input_command="$@" if [ "$mode" == vcheck ] ; then exec "${input_command}" - exit fi # @@ -343,5 +342,3 @@ if [ "$SPACK_DEBUG" = "TRUE" ]; then fi exec "${full_command[@]}" - - From 976ae91dccd36d1feb6d0d20cd1ef5b3470c4c39 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 14:11:57 +0100 Subject: [PATCH 149/339] llvm : removed duplicate version --- .../repos/builtin/packages/llvm/package.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a8f19f90711..934d994bd3e 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -196,25 +196,6 @@ class Llvm(Package): when='@%(version)s' % release, placement=resources[name].get('placement', None)) - # SVN - current develop - version('develop', svn='http://llvm.org/svn/llvm-project/llvm/trunk') - resource(name='clang', svn='http://llvm.org/svn/llvm-project/cfe/trunk', - destination='tools', when='@develop', placement='clang') - resource(name='compiler-rt', svn='http://llvm.org/svn/llvm-project/compiler-rt/trunk', - destination='projects', when='@develop', placement='compiler-rt') - resource(name='openmp', svn='http://llvm.org/svn/llvm-project/openmp/trunk', - destination='projects', when='@develop', placement='openmp') - resource(name='libcxx', svn='http://llvm.org/svn/llvm-project/libcxx/trunk', - destination='projects', when='@develop', placement='libcxx') - resource(name='libcxxabi', svn='http://llvm.org/svn/llvm-project/libcxxabi/trunk', - destination='projects', when='@develop', placement='libcxxabi') - resource(name='polly', svn='http://llvm.org/svn/llvm-project/polly/trunk', - destination='tools', when='@develop', placement='polly') - resource(name='lldb', svn='http://llvm.org/svn/llvm-project/lldb/trunk', - destination='tools', when='@develop', placement='lldb') - - - def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] From 6f42dd556d1ead8e7cad9788004dc33c11240564 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 24 Feb 2016 17:37:58 +0100 Subject: [PATCH 150/339] stage : on-going refactoring --- lib/spack/spack/package.py | 149 +++++++++++++++++----------------- lib/spack/spack/stage.py | 68 ++++++++++------ lib/spack/spack/test/stage.py | 122 +++++++++++----------------- 3 files changed, 167 insertions(+), 172 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..4fdc582479d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -847,93 +847,94 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s." % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - self.install(self.spec, self.prefix) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # Ensure that something was actually installed. - self._sanity_check_install() + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - install(log_path, log_install_path) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + def real_work(): + try: + tty.msg("Building %s." % self.name) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + self.install(self.spec, self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise + # Ensure that something was actually installed. + self._sanity_check_install() - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + install(log_path, log_install_path) - # Do the build. - spack.build_environment.fork(self, real_work) + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) + + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..96b1eaf3f2c 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -96,12 +96,44 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + + self.delete_on_exit = True + + def __enter__(self): + """ + Entering a stage context will create the stage directory + """ + # FIXME : if _setup is used only here, then it makes no sense to retain the function self._setup() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _cleanup_dead_links(self): """Remove any dead links in the stage directory.""" @@ -163,35 +195,17 @@ def _setup(self): mkdirp(spack.stage_path) self._cleanup_dead_links() - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - # If this is a temporary stage, them make the temp directory - tmp_dir = None if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in + if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) + os.symlink(tmp_dir, self.path) # if we're not using a tmp dir, create the stage directly in the # stage dir, rather than linking to it. else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) + if self._need_to_create_path(): + mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -389,6 +403,12 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + return self[0].__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self[0].__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..a7314eba4c5 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -192,116 +192,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) From 20845a739f15190cf9610da4375d3e3c9fc61b6b Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 24 Feb 2016 22:18:51 -0500 Subject: [PATCH 151/339] Libevent depends on OpenSSL --- var/spack/repos/builtin/packages/libevent/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 11b1083d672..2a44c49325d 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -23,6 +23,9 @@ class Libevent(Package): version('2.0.12', '42986228baf95e325778ed328a93e070') + depends_on('openssl') + + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 30d9ca2bde033da28b6f6105f93ee05f4fd0acfa Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Fri, 26 Feb 2016 16:06:17 -0800 Subject: [PATCH 152/339] Updated the silo package and added the '+fortran' variant. --- .../repos/builtin/packages/silo/package.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 9eda11df15f..d1aed78e0e8 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -1,19 +1,28 @@ from spack import * class Silo(Package): - """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" + """Silo is a library for reading and writing a wide variety of scientific + data to binary, disk files.""" homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" - #version('4.9', 'a83eda4f06761a86726e918fc55e782a') version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') - depends_on("hdf5@:1.8.12") + variant('fortran', default=True, description='Enable Fortran support') + + depends_on("hdf5") def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-hdf5=%s" %spec['hdf5'].prefix) + config_args = [ + '--enable-fortran' if '+fortran' in spec else '--disable-fortran', + ] + + configure( + "--prefix=%s" % prefix, + "--with-hdf5=%s,%s" % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), + "--with-zlib=%s,%s" % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + *config_args) make() make("install") From 8f3ac9ac8b7b1d16672e7a52f691966de1f8483f Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Sat, 27 Feb 2016 01:40:32 +0100 Subject: [PATCH 153/339] adding new version to praview, compiled with spack -d install -j 8 --keep-stage paraview@5.0.0+qt+python+tcl+opengl2%gcc@4.8.2 ^netcdf -mpi --- .../repos/builtin/packages/paraview/package.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index e43bdd44930..ccf2d14c06e 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -2,9 +2,11 @@ class Paraview(Package): homepage = 'http://www.paraview.org' - url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + url = 'http://www.paraview.org/files/v5.0/ParaView-v' + _url_str = 'http://www.paraview.org/files/v%s/ParaView-v%s-source.tar.gz' - version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378') + version('5.0.0', '4598f0b421460c8bbc635c9a1c3bdbee') variant('python', default=False, description='Enable Python support') @@ -25,8 +27,8 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5') depends_on('hdf5+mpi', when='+mpi') + depends_on('hdf5~mpi', when='~mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') @@ -35,6 +37,11 @@ class Paraview(Package): #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') + + def url_for_version(self, version): + """Handle ParaView version-based custom URLs.""" + return self._url_str % (version.up_to(2), version) + def install(self, spec, prefix): with working_dir('spack-build', create=True): From 67f327f805868d369eec0392631392281b68c39d Mon Sep 17 00:00:00 2001 From: Ian Lee Date: Sun, 28 Feb 2016 19:47:19 -0800 Subject: [PATCH 154/339] Updated links to use new llnl.gov address --- README.md | 4 ++-- lib/spack/docs/getting_started.rst | 2 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 2 +- lib/spack/spack/resource.py | 2 +- lib/spack/spack/test/namespace_trie.py | 2 +- lib/spack/spack/test/tally_plugin.py | 8 ++++---- var/spack/repos/builtin/packages/gdb/package.py | 2 +- var/spack/repos/builtin/packages/texinfo/package.py | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bdce345764f..8664953c0cc 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://llnl.github.io/spack/features.html) +[Feature Overview](http://software.llnl.gov/spack/features.html) for examples and highlights. To install spack and install your first package: @@ -31,7 +31,7 @@ To install spack and install your first package: Documentation ---------------- -[**Full documentation**](http://llnl.github.io/spack) for Spack is +[**Full documentation**](http://software.llnl.gov/spack) for Spack is the first place to look. See also: diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 67ca18e71ae..2c5b68ea650 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -`_: +`_: .. code-block:: sh diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 34c755fb67f..908f5400ab2 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index f58cd52125b..e8d0cc09ec2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2bf92947fd3..ddfaaf4cb09 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://scalability-software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ def __init__(self): self.successCount = 0 self.failCount = 0 self.errorCount = 0 - + @property def numberOfTestsRun(self): """Excludes skipped tests""" @@ -48,10 +48,10 @@ def configure(self, options, conf): def addSuccess(self, test): self.successCount += 1 - + def addError(self, test, err): self.errorCount += 1 - + def addFailure(self, test, err): self.failCount += 1 diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index dd02b426b94..b346fe80c28 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/repos/builtin/packages/texinfo/package.py b/var/spack/repos/builtin/packages/texinfo/package.py index a83c10c0c19..6cf8d790722 100644 --- a/var/spack/repos/builtin/packages/texinfo/package.py +++ b/var/spack/repos/builtin/packages/texinfo/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify From a339ac0a727458ab55b2e0d27ff17dcc139c0a32 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:29:30 -0500 Subject: [PATCH 155/339] Bug Fix: When Spack create roots around for other versions, it sometimes finds files it thinks are tarballs, but are not. Previously, it would crash if any such files are found. This change allows it to simply skip them and move on, processing the rest of the files it finds correctly. --- lib/spack/spack/cmd/checksum.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b1ad89dbb8a..c4519932335 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs): tty.msg("Downloading...") hashes = [] - for i, (url, version) in enumerate(zip(urls, versions)): + i = 0 + for url, version in zip(urls, versions): stage = Stage(url) try: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) - hashes.append( - spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) except FailedDownloadError, e: tty.msg("Failed to fetch %s" % url) continue + except Exception, e: + tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + continue finally: if not keep_stage: stage.destroy() + i += 1 - return zip(versions, hashes) + return hashes From e414c5fdfbc91e5934ee59233475f8f862e2e0ce Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:31:18 -0500 Subject: [PATCH 156/339] Added missing cmake dependencies --- var/spack/repos/builtin/packages/cgal/package.py | 1 + var/spack/repos/builtin/packages/curl/package.py | 6 +++--- var/spack/repos/builtin/packages/expat/package.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/cgal/package.py b/var/spack/repos/builtin/packages/cgal/package.py index 97356433be1..ef4a2736dbb 100644 --- a/var/spack/repos/builtin/packages/cgal/package.py +++ b/var/spack/repos/builtin/packages/cgal/package.py @@ -46,6 +46,7 @@ class Cgal(Package): depends_on('mpfr') depends_on('gmp') depends_on('zlib') + depends_on('cmake') # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL) # FIXME : Optional third party libraries missing diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c71..6c302281a77 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') - depends_on("openssl") +# depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix, - '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix) +# '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") diff --git a/var/spack/repos/builtin/packages/expat/package.py b/var/spack/repos/builtin/packages/expat/package.py index 082da5bf0be..3f925c65461 100644 --- a/var/spack/repos/builtin/packages/expat/package.py +++ b/var/spack/repos/builtin/packages/expat/package.py @@ -7,6 +7,7 @@ class Expat(Package): version('2.1.0', 'dd7dab7a5fea97d2a6a43f511449b7cd') + depends_on('cmake') def install(self, spec, prefix): From 15ae92aae9c8ac521bb5d7ee5f210d265494d87a Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Mon, 29 Feb 2016 22:59:28 -0500 Subject: [PATCH 157/339] New packages added: blitz, netcdf-cxx4, netcdf-fortran, proj, udunits2 --- .../repos/builtin/packages/blitz/package.py | 40 ++++++++++++++++++ .../builtin/packages/netcdf-cxx4/package.py | 41 +++++++++++++++++++ .../packages/netcdf-fortran/package.py | 21 ++++++++++ .../repos/builtin/packages/proj/package.py | 37 +++++++++++++++++ .../builtin/packages/udunits2/package.py | 16 ++++++++ 5 files changed, 155 insertions(+) create mode 100644 var/spack/repos/builtin/packages/blitz/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-cxx4/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf-fortran/package.py create mode 100644 var/spack/repos/builtin/packages/proj/package.py create mode 100644 var/spack/repos/builtin/packages/udunits2/package.py diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py new file mode 100644 index 00000000000..82ff6349250 --- /dev/null +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -0,0 +1,40 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install blitz +# +# You can always get back here to change things with: +# +# spack edit blitz +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Blitz(Package): + """N-dimensional arrays for C++""" + homepage = "http://github.com/blitzpp/blitz" + +# This version doesn't have the configure script generated yet. + url = "https://github.com/blitzpp/blitz/tarball/1.0.0" +#http://prdownloads.sourceforge.net/%(namelower)s + + version('1.0.0', '9f040b9827fe22228a892603671a77af') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + # FIXME: Spack couldn't guess one, so here are some options: + configure('--prefix=%s' % prefix) + # cmake('.', *std_cmake_args) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py new file mode 100644 index 00000000000..9d70eab05f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -0,0 +1,41 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install netcdf-cxx4 +# +# You can always get back here to change things with: +# +# spack edit netcdf-cxx4 +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class NetcdfCxx4(Package): + """C++ interface for NetCDF4""" + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" + + version('4.2', 'd019853802092cf686254aaba165fc81') + + + variant('mpi', default=True, description='Enables MPI parallelism') +# variant('hdf4', default=False, description="Enable HDF4 support") + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py new file mode 100644 index 00000000000..8e5c8ecc3a0 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -0,0 +1,21 @@ +from spack import * + +class NetcdfFortran(Package): + """Fortran interface for NetCDF4""" + + homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" + + version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') + + variant('mpi', default=True, description='Enables MPI parallelism') + + # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) + depends_on('mpi', when='+mpi') + depends_on('netcdf') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py new file mode 100644 index 00000000000..4a0d3feac75 --- /dev/null +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -0,0 +1,37 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install proj +# +# You can always get back here to change things with: +# +# spack edit proj +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Proj(Package): + """Cartographic Projections""" + homepage = "https://github.com/OSGeo/proj.4/wiki" + url = "http://download.osgeo.org/proj/proj-4.9.2.tar.gz" + + version('4.9.2', '9843131676e31bbd903d60ae7dc76cf9') + version('4.9.1', '3cbb2a964fd19a496f5f4265a717d31c') + version('4.8.0', 'd815838c92a29179298c126effbb1537') + version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') + version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/udunits2/package.py b/var/spack/repos/builtin/packages/udunits2/package.py new file mode 100644 index 00000000000..9954a733bb5 --- /dev/null +++ b/var/spack/repos/builtin/packages/udunits2/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Udunits2(Package): + """Automated units conversion""" + + homepage = "http://www.unidata.ucar.edu/software/udunits" + url = "ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.2.20.tar.gz" + + version('2.2.20', '1586b70a49dfe05da5fcc29ef239dce0') + + depends_on('expat') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From eb1d90a8cfadeb556754b57e5d21e5605b9a0e9d Mon Sep 17 00:00:00 2001 From: citibeth Date: Mon, 29 Feb 2016 23:31:54 -0500 Subject: [PATCH 158/339] Undid accidental change on this branch. --- var/spack/repos/builtin/packages/curl/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 6c302281a77..9e684445c71 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -13,13 +13,13 @@ class Curl(Package): version('7.43.0', '11bddbb452a8b766b932f859aaeeed39') version('7.42.1', '296945012ce647b94083ed427c1877a8') -# depends_on("openssl") + depends_on("openssl") depends_on("zlib") def install(self, spec, prefix): configure('--prefix=%s' % prefix, - '--with-zlib=%s' % spec['zlib'].prefix) -# '--with-ssl=%s' % spec['openssl'].prefix) + '--with-zlib=%s' % spec['zlib'].prefix, + '--with-ssl=%s' % spec['openssl'].prefix) make() make("install") From a9f0b98d4d30e44032f5b9c86c5b6e27425098f5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 1 Mar 2016 15:53:53 -0600 Subject: [PATCH 159/339] Typo fix --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index aacba996b39..a323c481243 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -130,7 +130,7 @@ if [ -z "$mode" ]; then done fi -# Dump the version and exist if we're in testing mode. +# Dump the version and exit if we're in testing mode. if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit From 8174489787c56cee1726ca36799c236e4869f471 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 1 Mar 2016 15:25:57 -0700 Subject: [PATCH 160/339] + Provide two new variants for cmake: 1) +qt - build the cmake-gui Qt application. - adds a dependency on Qt. 2) +sphinxbuild - build the html CMake documentation. - adds a dependency on python and py-sphinx --- .../repos/builtin/packages/cmake/package.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index e20c1e4aeb0..f39a6812843 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -37,16 +37,34 @@ class Cmake(Package): version('2.8.10.2', '097278785da7182ec0aea8769d06860c') variant('ncurses', default=True, description='Enables the build of the ncurses gui') + variant('qt', default=False, description='Enables the build of cmake-gui') + variant('sphinxbuild', default=False, description='Enables the generation of html and man page documentation') + depends_on('ncurses', when='+ncurses') + depends_on('qt', when='+qt') + depends_on('python@2.7.11:', when='+sphinxbuild') + depends_on('py-sphinx', when='+sphinxbuild') def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) - def install(self, spec, prefix): - configure('--prefix=' + prefix, - '--parallel=' + str(make_jobs), - '--', '-DCMAKE_USE_OPENSSL=ON') + + options = ['--prefix=%s' % prefix] + options.append('--parallel=%s' % str(make_jobs)) + + if '+qt' in spec: + options.append('--qt-gui') + + if '+sphinxbuild' in spec: + options.append('--sphinx-html') + options.append('--sphinx-man') + + options.append('--') + options.append('-DCMAKE_USE_OPENSSL=ON') + + configure(*options) + make() make('install') From c488f7c4d8e2ff240d561820df11a51518199a53 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 1 Mar 2016 23:57:34 -0800 Subject: [PATCH 161/339] Fix bug in install permission setting. --- lib/spack/llnl/util/filesystem.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -152,15 +152,20 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) + + # Expand dsst to its eventual full path if it is a directory. + if os.path.isdir(dest): + dest = join_path(dest, os.path.basename(src)) + shutil.copy(src, dest) set_install_permissions(dest) copy_mode(src, dest) From be306d09e99a69732afc3f44724222ab6c6d71cc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:04:46 -0800 Subject: [PATCH 162/339] Move repo creation code into repository.py --- lib/spack/spack/cmd/repo.py | 46 +-------------------------- lib/spack/spack/repository.py | 58 ++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 908f5400ab2..c2e352786d1 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -74,51 +74,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - root = canonicalize_path(args.directory) - namespace = args.namespace - - if not args.namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - tty.die('File %s already exists and is not a directory' % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - tty.die('Cannot create new repo in %s: cannot access directory.' % root) - if os.listdir(root): - tty.die('Cannot create new repo in %s: directory is not empty.' % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - tty.die("Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - tty.die('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - + full_path, namespace = create_repo(args.directory, args.namespace) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", 'spack repo add %s' % full_path) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index e8d0cc09ec2..6aa75cb43e5 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -33,7 +33,7 @@ from external import yaml import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import * import spack.error import spack.config @@ -705,6 +705,58 @@ def __contains__(self, pkg_name): return self.exists(pkg_name) +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and the namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" @@ -713,6 +765,10 @@ class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + class BadRepoError(RepoError): """Raised when repo layout is invalid.""" From 21d125c9147f5d3cfc0eb8a9516c8ce24df7279b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:08:36 -0800 Subject: [PATCH 163/339] Fixes #175: Dump environment provenance as well as build log. --- lib/spack/spack/directory_layout.py | 14 +++++++++++++- lib/spack/spack/package.py | 7 +++++++ lib/spack/spack/util/environment.py | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 3e416a6a1fc..29d87b65b33 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -173,7 +173,9 @@ def __init__(self, root, **kwargs): self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.out' # TODO: use config file. + self.build_log_name = 'build.out' # build log. + self.build_env_name = 'build.env' # build environment + self.packages_dir = 'repos' # archive of package.py files # Cache of already written/read extension maps. self._extension_maps = {} @@ -231,6 +233,16 @@ def build_log_path(self, spec): self.build_log_name) + def build_env_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.build_env_name) + + + def build_packages_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.packages_dir) + + def create_install_directory(self, spec): _check_concrete(spec) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8019b29cba5..5b1927fe8f4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -66,6 +66,7 @@ from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension from spack.util.executable import ProcessError +from spack.util.environment import dump_environment """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -884,10 +885,14 @@ def real_work(): # Do the real install in the source directory. self.stage.chdir_to_source() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + # This redirects I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) self.install(self.spec, self.prefix) # Ensure that something was actually installed. @@ -896,7 +901,9 @@ def real_work(): # Move build log into install directory on success if not fake: log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) install(log_path, log_install_path) + install(env_path, env_install_path) # On successful install, remove the stage. if not keep_stage: diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index cd413dcfbc8..ae8e5708bef 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys): for key in keys: if key in dictionary: dictionary.pop(key) + + +def dump_environment(path): + """Dump the current environment out to a file.""" + with open(path, 'w') as env_file: + for key,val in sorted(os.environ.items()): + env_file.write("%s=%s\n" % (key, val)) From 52081c46d641fcf679e70a04e9937db67e8787e4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 00:09:24 -0800 Subject: [PATCH 164/339] Fixes #469: Store package.py files in the .spack directory. - Adds packages in spack repos inside the .spack directory, so that packages can be rebuilt automatically later. --- lib/spack/spack/package.py | 51 +++++++++++++++++++++++++++++++++++ lib/spack/spack/repository.py | 39 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5b1927fe8f4..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -58,6 +58,7 @@ import spack.mirror import spack.hooks import spack.directives +import spack.repository import spack.build_environment import spack.url import spack.util.web @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -905,6 +907,9 @@ def real_work(): install(log_path, log_install_path) install(env_path, env_install_path) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) + # On successful install, remove the stage. if not keep_stage: self.stage.destroy() @@ -1219,6 +1224,52 @@ def validate_package_url(url_string): tty.die("Invalid file type in URL: '%s'" % url_string) +def dump_packages(spec, path): + """Dump all package information for a spec and its dependencies. + + This creates a package repository within path for every + namespace in the spec DAG, and fills the repos wtih package + files and patch files for every node in the DAG. + """ + mkdirp(path) + + # Copy in package.py files from any dependencies. + # Note that we copy them in as they are in the *install* directory + # NOT as they are in the repository, because we want a snapshot of + # how *this* particular build was done. + for node in spec.traverse(): + if node is not spec: + # Locate the dependency package in the install tree and find + # its provenance information. + source = spack.install_layout.build_packages_path(node) + source_repo_root = join_path(source, node.namespace) + + # There's no provenance installed for the source package. Skip it. + # User can always get something current from the builtin repo. + if not os.path.isdir(source_repo_root): + continue + + # Create a source repo and get the pkg directory out of it. + try: + source_repo = spack.repository.Repo(source_repo_root) + source_pkg_dir = source_repo.dirname_for_package_name(node.name) + except RepoError as e: + tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + + # Create a destination repository + dest_repo_root = join_path(path, node.namespace) + if not os.path.exists(dest_repo_root): + spack.repository.create_repo(dest_repo_root) + repo = spack.repository.Repo(dest_repo_root) + + # Get the location of the package in the dest repo. + dest_pkg_dir = repo.dirname_for_package_name(node.name) + if node is not spec: + install_tree(source_pkg_dir, dest_pkg_dir) + else: + spack.repo.dump_provenance(node, dest_pkg_dir) + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 6aa75cb43e5..8d06fefe7f0 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -316,6 +316,16 @@ def get(self, spec, new=False): return self.repo_for_pkg(spec).get(spec) + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) @@ -552,6 +562,35 @@ def get(self, spec, new=False): return self._instances[key] + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): """Clear entire package instance cache.""" self._instances.clear() From a0c6519de9d03744e33b5187d42ec0d966dbd3e0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 01:41:43 -0800 Subject: [PATCH 165/339] Fixes #476: create was broken for FTP URLs. --- lib/spack/spack/util/web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index e26daef2964..73f4858b023 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,12 +86,12 @@ def _spider(args): if not "Content-type" in resp.headers: tty.debug("ignoring page " + url) - return pages + return pages, links if not resp.headers["Content-type"].startswith('text/html'): tty.debug("ignoring page " + url + " with content type " + resp.headers["Content-type"]) - return pages + return pages, links # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -173,7 +173,7 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) return pages, links From 726b350689bf6da7b82eec170001adc8100beb71 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 12:52:38 +0100 Subject: [PATCH 166/339] test : fixed failing unit tests --- lib/spack/spack/mirror.py | 49 +++++++++---------- lib/spack/spack/stage.py | 7 ++- lib/spack/spack/test/configure_guess.py | 12 ++--- lib/spack/spack/test/git_fetch.py | 31 +++++++------ lib/spack/spack/test/hg_fetch.py | 31 +++++++------ lib/spack/spack/test/link_tree.py | 6 +-- lib/spack/spack/test/mirror.py | 62 ++++++++++++------------- lib/spack/spack/test/svn_fetch.py | 31 +++++++------ 8 files changed, 116 insertions(+), 113 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..bc8870926f9 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -168,32 +168,33 @@ def create(path, specs, **kwargs): pkg = spec.package tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) try: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + with pkg.stage: + for ii, stage in enumerate(pkg.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + everything_already_exists = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) if everything_already_exists: present.append(spec) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 96b1eaf3f2c..956d1c87069 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -404,10 +404,13 @@ def path(self): return self[0].path def __enter__(self): - return self[0].__enter__() + for item in self: + item.__enter__() + return self def __exit__(self, exc_type, exc_val, exc_tb): - return self[0].__exit__(exc_type, exc_val, exc_tb) + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): return self[0].chdir_to_source() diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..bc2332acc23 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -52,8 +52,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +62,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..caa076823e4 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -76,26 +76,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..75aa7ab17e7 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -68,26 +68,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..7b67e873dde 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -38,6 +38,8 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ + self.stage.__enter__() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +53,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.__exit__(None, None, None) def check_file_link(self, filename): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..9e2c6319782 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -74,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +97,36 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) + saved_checksum_setting = spack.do_checksum + with pkg.stage: + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) + dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..6ac9e2f3431 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -82,26 +82,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): From 65b2a24f7c12380d0815e5705d1c1e66e5fd22a9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:03:40 +0100 Subject: [PATCH 167/339] stage : removed _setup method --- lib/spack/spack/stage.py | 57 ++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 956d1c87069..48770fb4079 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -111,9 +111,32 @@ def __init__(self, url_or_fetch_strategy, **kwargs): def __enter__(self): """ Entering a stage context will create the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. """ - # FIXME : if _setup is used only here, then it makes no sense to retain the function - self._setup() + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a temporary stage, them make the temp directory + if self.tmp_root: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -180,36 +203,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a temporary stage, them make the temp directory - if self.tmp_root: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" From d649b715edae0871254695e2deacf078554a1475 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 13:16:04 +0100 Subject: [PATCH 168/339] stage : updated functions doc _cleanup_dead_links : fixed minor bug --- lib/spack/spack/stage.py | 41 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 48770fb4079..900acd664da 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -164,7 +157,7 @@ def _cleanup_dead_links(self): path = join_path(spack.stage_path, file) if os.path.islink(path): real_path = os.path.realpath(path) - if not os.path.exists(path): + if not os.path.exists(real_path): os.unlink(path) def _need_to_create_path(self): From 9001b9ed3c01f8ccaceaca60d6a34c3551f77240 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 15:56:09 +0100 Subject: [PATCH 169/339] package : minor syntax fixes mirror : extracted add_single_spec from create --- lib/spack/spack/mirror.py | 98 ++++++++++++++++++++------------------ lib/spack/spack/package.py | 4 +- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index bc8870926f9..5ed7aff176b 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,58 +157,65 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - with pkg.stage: - for ii, stage in enumerate(pkg.stage): - fetcher = stage.fetcher - if ii == 0: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) - name = spec.format("$_$@") - else: - resource = stage.resource - archive_path = join_path(subdir, suggest_archive_basename(resource)) - name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) + add_single_spec(spec, mirror_root, categories, **kwargs) - if os.path.exists(archive_path): - tty.msg("{name} : already added".format(name=name)) - else: - everything_already_exists = False - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("{name} : checksum passed".format(name=name)) + return categories['present'], categories['mirrored'], categories['error'] - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) - return (present, mirrored, error) + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + spec_exists_in_mirror = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) + + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 4fdc582479d..29b781a7499 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -732,7 +732,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -912,7 +912,7 @@ def real_work(): % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) - except ProcessError, e: + except ProcessError as e: # Annotate with location of build log. e.build_log = log_path cleanup() From 901e4851b993f73c4bdd92b85b2469d6673b7617 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:19:32 +0100 Subject: [PATCH 170/339] _cleanup_dead_links : factored method into a function and put it in llnl.filesystem --- lib/spack/llnl/util/filesystem.py | 19 +++++++++++++++++-- lib/spack/spack/stage.py | 20 ++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde719..4a708b639a3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] import os import sys @@ -235,7 +235,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -339,3 +339,18 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) \ No newline at end of file diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 900acd664da..e9106431920 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -98,7 +98,7 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) - + # Flag to decide whether to delete the stage folder on exit or not self.delete_on_exit = True def __enter__(self): @@ -113,20 +113,17 @@ def __enter__(self): """ # Create the top-level stage directory mkdirp(spack.stage_path) - self._cleanup_dead_links() + remove_dead_links(spack.stage_path) - # If this is a temporary stage, them make the temp directory + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path if self.tmp_root: if self._need_to_create_path(): tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. else: if self._need_to_create_path(): mkdirp(self.path) - # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -151,15 +148,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if self.delete_on_exit: self.destroy() - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. From 4d63544fe918393de9265f8879bb0199a542cc3b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 16:55:57 +0100 Subject: [PATCH 171/339] remove_link_tree : moved to llnl.util.filesystem --- lib/spack/llnl/util/filesystem.py | 20 ++++++++++++++++++-- lib/spack/spack/stage.py | 21 +++------------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 4a708b639a3..015eeb9aa1c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -353,4 +353,20 @@ def remove_dead_links(root): if os.path.islink(path): real_path = os.path.realpath(path) if not os.path.exists(real_path): - os.unlink(path) \ No newline at end of file + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e9106431920..e87b822a8f4 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -114,15 +114,13 @@ def __enter__(self): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, # otherwise create the stage directory in self.path - if self.tmp_root: - if self._need_to_create_path(): + if self._need_to_create_path(): + if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) os.symlink(tmp_dir, self.path) - else: - if self._need_to_create_path(): + else: mkdirp(self.path) # Make sure we can actually do something with the stage we made. ensure_access(self.path) @@ -436,19 +434,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): From ca41909ec52ab2b8f82ca91421d3882300531ac2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:19:27 +0100 Subject: [PATCH 172/339] package : removed do_clean() --- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 9 +------- lib/spack/spack/test/git_fetch.py | 4 ---- lib/spack/spack/test/hg_fetch.py | 3 --- lib/spack/spack/test/mirror.py | 38 ++++++++++++------------------- lib/spack/spack/test/svn_fetch.py | 4 ---- 6 files changed, 17 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 29b781a7499..fa64b2c0472 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1138,13 +1138,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1181,7 +1174,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index caa076823e4..6d6a67a1d38 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -52,19 +52,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 75aa7ab17e7..d884ed78a0f 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -49,13 +49,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 9e2c6319782..f117e042424 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -104,29 +104,21 @@ def check_mirror(self): saved_checksum_setting = spack.do_checksum with pkg.stage: - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 6ac9e2f3431..13a00f5df79 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -51,13 +51,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +67,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with From 21cd05aad5e8f45a7ebac7b213855a16685d229f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 2 Mar 2016 17:28:21 +0100 Subject: [PATCH 173/339] package : updated doc --- lib/spack/spack/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fa64b2c0472..0214dcd7716 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -291,7 +291,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. From 6bbadbfaac406f9abe81e64f174d8a2b4c9c255e Mon Sep 17 00:00:00 2001 From: Glenn Johnson Date: Wed, 2 Mar 2016 13:18:43 -0600 Subject: [PATCH 174/339] Add CPATH pointing to include directories. --- lib/spack/docs/basic_usage.rst | 2 +- lib/spack/spack/modules.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db6..f94ac3d2ba8 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -896,7 +896,7 @@ Or, similarly with modules, you could type: $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use a package, you can type unload or unuse similarly: .. code-block:: sh diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c8347635646..c27043db8cd 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -33,6 +33,7 @@ * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH @@ -121,6 +122,7 @@ def add_path(path_name, directory): ('LIBRARY_PATH', self.spec.prefix.lib64), ('LD_LIBRARY_PATH', self.spec.prefix.lib), ('LD_LIBRARY_PATH', self.spec.prefix.lib64), + ('CPATH', self.spec.prefix.include), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: From 976d0240c467b9ff74333fafcd2ac1e896819a80 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 2 Mar 2016 18:58:47 -0500 Subject: [PATCH 175/339] Removed FIXMEs. --- .../repos/builtin/packages/blitz/package.py | 27 +------------------ .../builtin/packages/netcdf-cxx4/package.py | 21 +-------------- .../packages/netcdf-fortran/package.py | 2 +- .../repos/builtin/packages/proj/package.py | 19 +------------ 4 files changed, 4 insertions(+), 65 deletions(-) diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py index 82ff6349250..9413b276fe9 100644 --- a/var/spack/repos/builtin/packages/blitz/package.py +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -1,40 +1,15 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install blitz -# -# You can always get back here to change things with: -# -# spack edit blitz -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Blitz(Package): """N-dimensional arrays for C++""" homepage = "http://github.com/blitzpp/blitz" - -# This version doesn't have the configure script generated yet. url = "https://github.com/blitzpp/blitz/tarball/1.0.0" -#http://prdownloads.sourceforge.net/%(namelower)s version('1.0.0', '9f040b9827fe22228a892603671a77af') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. - # FIXME: Spack couldn't guess one, so here are some options: configure('--prefix=%s' % prefix) - # cmake('.', *std_cmake_args) - - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 9d70eab05f5..8d51a106794 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install netcdf-cxx4 -# -# You can always get back here to change things with: -# -# spack edit netcdf-cxx4 -# -# See the spack documentation for more information on building -# packages. -# from spack import * class NetcdfCxx4(Package): @@ -25,17 +9,14 @@ class NetcdfCxx4(Package): variant('mpi', default=True, description='Enables MPI parallelism') -# variant('hdf4', default=False, description="Enable HDF4 support") - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure('--prefix=%s' % prefix) - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 8e5c8ecc3a0..9e4aee95fb8 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,7 +10,7 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # NetCDF-CXX4 doesn't really depend (directly) on MPI. However... this + # netcdf-fortran doesn't really depend (directly) on MPI. However... this # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) depends_on('mpi', when='+mpi') depends_on('netcdf') diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py index 4a0d3feac75..797772f4f6f 100644 --- a/var/spack/repos/builtin/packages/proj/package.py +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -1,19 +1,3 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install proj -# -# You can always get back here to change things with: -# -# spack edit proj -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Proj(Package): @@ -27,8 +11,7 @@ class Proj(Package): version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") + # No dependencies def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 717bcb862428a1b1a3b9c70f925c3fe6c325f622 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:17:49 +0100 Subject: [PATCH 176/339] test : optimized import statements --- lib/spack/spack/test/concretize.py | 2 -- lib/spack/spack/test/config.py | 6 ++--- lib/spack/spack/test/configure_guess.py | 9 ++------ lib/spack/spack/test/database.py | 11 +++++---- lib/spack/spack/test/directory_layout.py | 13 +++++------ lib/spack/spack/test/git_fetch.py | 11 ++------- lib/spack/spack/test/hg_fetch.py | 10 +++------ lib/spack/spack/test/install.py | 9 ++------ lib/spack/spack/test/link_tree.py | 7 ++---- lib/spack/spack/test/lock.py | 8 +++---- lib/spack/spack/test/make_executable.py | 6 ++--- lib/spack/spack/test/mirror.py | 5 ++--- lib/spack/spack/test/mock_packages_test.py | 8 +++---- lib/spack/spack/test/mock_repo.py | 4 ---- lib/spack/spack/test/multimethod.py | 5 +---- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +--- lib/spack/spack/test/packages.py | 26 ++++++---------------- lib/spack/spack/test/python_version.py | 3 +-- lib/spack/spack/test/spec_dag.py | 2 -- lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 ++- lib/spack/spack/test/stage.py | 8 +++---- lib/spack/spack/test/svn_fetch.py | 14 ++++-------- lib/spack/spack/test/tally_plugin.py | 4 ++-- lib/spack/spack/test/unit_install.py | 3 ++- lib/spack/spack/test/url_extrapolate.py | 3 --- lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 31 files changed, 62 insertions(+), 129 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index bc2332acc23..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 6d6a67a1d38..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index d884ed78a0f..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 7b67e873dde..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,8 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - # FIXME : possibly this test needs to be refactored to avoid the explicit call to __enter__ and __exit__ - self.stage.__enter__() + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -54,7 +51,7 @@ def setUp(self): self.link_tree = LinkTree(source_path) def tearDown(self): - self.stage.__exit__(None, None, None) + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f117e042424..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d5..83fb34bf765 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..07ff0b21af0 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): @@ -77,30 +75,20 @@ def test_package_class_names(self): # def test_import_package(self): - import spack.pkg.builtin.mock.mpich + pass def test_import_package_as(self): - import spack.pkg.builtin.mock.mpich as mp + pass def test_import_class_from_package(self): - from spack.pkg.builtin.mock.mpich import Mpich + pass def test_import_module_from_package(self): - from spack.pkg.builtin.mock import mpich + pass def test_import_namespace_container_modules(self): - import spack.pkg - import spack.pkg as p - from spack import pkg - - import spack.pkg.builtin - import spack.pkg.builtin as b - from spack.pkg import builtin - - import spack.pkg.builtin.mock - import spack.pkg.builtin.mock as m - from spack.pkg.builtin import mock + pass diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index a7314eba4c5..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 13a00f5df79..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47ce..eb1e4a3240e 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From 353d12d2e6e24c0f62795dd9e649a801b566c4d8 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 09:18:47 +0100 Subject: [PATCH 177/339] stage : added create function as decided in #477 comments --- lib/spack/spack/stage.py | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index e87b822a8f4..5d8b09fa9e6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -105,26 +105,10 @@ def __enter__(self): """ Entering a stage context will create the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Returns: + self """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - + self.create() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -314,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory From 5ef36215569e1151164ecb630011de425387732a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 10:10:30 +0100 Subject: [PATCH 178/339] stage : minor syntax fix --- lib/spack/spack/stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5d8b09fa9e6..985043c1035 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -246,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue From db92699c02f9aaf435d6bc073c89cd2c75d8824c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 3 Mar 2016 11:05:36 +0100 Subject: [PATCH 179/339] test : fixed tests that were broken by import optimization --- lib/spack/spack/test/packages.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 07ff0b21af0..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -75,20 +75,30 @@ def test_package_class_names(self): # def test_import_package(self): - pass + import spack.pkg.builtin.mock.mpich def test_import_package_as(self): - pass + import spack.pkg.builtin.mock.mpich as mp def test_import_class_from_package(self): - pass + from spack.pkg.builtin.mock.mpich import Mpich def test_import_module_from_package(self): - pass + from spack.pkg.builtin.mock import mpich def test_import_namespace_container_modules(self): - pass + import spack.pkg + import spack.pkg as p + from spack import pkg + + import spack.pkg.builtin + import spack.pkg.builtin as b + from spack.pkg import builtin + + import spack.pkg.builtin.mock + import spack.pkg.builtin.mock as m + from spack.pkg.builtin import mock From a531a4f5e306388fcb6aa3c2f9db7bb93b530242 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Thu, 3 Mar 2016 18:00:50 +0100 Subject: [PATCH 180/339] help spack versions paraview to get versions without v --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e260..ad551a6dedb 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -225,7 +225,7 @@ def parse_version_offset(path): (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem), # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz - (r'-([^-]+(-alpha|-beta)?)', stem), + (r'-v?([^-]+(-alpha|-beta)?)', stem), # e.g. astyle_1.23_macosx.tar.gz (r'_([^_]+(_alpha|_beta)?)', stem), From 7183db1b7df3bce7e5f174ce38a50476c2e3b218 Mon Sep 17 00:00:00 2001 From: Scott Pakin Date: Thu, 3 Mar 2016 11:30:57 -0700 Subject: [PATCH 181/339] Added LLVM 3.7.1 support --- .../repos/builtin/packages/llvm/package.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 934d994bd3e..280e400f696 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -34,7 +34,7 @@ class Llvm(Package): it is the full name of the project. """ homepage = 'http://llvm.org/' - url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + url = 'http://llvm.org/releases/3.7.1/llvm-3.7.1.src.tar.xz' version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package @@ -132,6 +132,21 @@ class Llvm(Package): 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', } }, + { + 'version' : '3.7.1', + 'md5':'bf8b3a2c79e61212c5409041dfdbd319', + 'resources' : { + 'compiler-rt' : '1c6975daf30bb3b0473b53c3a1a6ff01', + 'openmp' : 'b4ad08cda4e5c22e42b66062b140438e', + 'polly' : '3a2a7367002740881637f4d47bca4dc3', + 'libcxx' : 'f9c43fa552a10e14ff53b94d04bea140', + 'libcxxabi' : '52d925afac9f97e9dcac90745255c169', + 'clang' : '0acd026b5529164197563d135a8fd83e', + 'clang-tools-extra' : '5d49ff745037f061a7c86aeb6a24c3d2', + 'lldb' : 'a106d8a0d21fc84d76953822fbaf3398', + 'llvm-libunwind' : '814bd52c9247c5d04629658fbcb3ab8c', + } + }, { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', From c5d9ee89246b3d2aeddb756a04588424051d3295 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 3 Mar 2016 14:41:20 -0800 Subject: [PATCH 182/339] Revert "refactoring proposal : turn Stage into a context manager" --- lib/spack/llnl/util/filesystem.py | 35 +--- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 176 +++++++++---------- lib/spack/spack/stage.py | 186 +++++++++++---------- lib/spack/spack/test/concretize.py | 2 + lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 ++- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 +++-- lib/spack/spack/test/hg_fetch.py | 44 ++--- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 + lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 - lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 + lib/spack/spack/test/spec_semantics.py | 1 + lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++++------ lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 + lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 + lib/spack/spack/test/versions.py | 1 - lib/spack/spack/test/yaml.py | 1 - 36 files changed, 504 insertions(+), 423 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..da3cf96050f 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError as e: + except OSError, e: os.remove(dest) os.symlink(src, dest) @@ -344,34 +344,3 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) - -def remove_dead_links(root): - """ - Removes any dead link that is present in root - - Args: - root: path where to search for dead links - - """ - for file in os.listdir(root): - path = join_path(root, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(real_path): - os.unlink(path) - -def remove_linked_tree(path): - """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. - - Args: - path: directory to be removed - - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 5ed7aff176b..fa29e208036 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,6 +110,7 @@ def suggest_archive_basename(resource): return basename + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -157,29 +158,17 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - categories = { - 'present': [], - 'mirrored': [], - 'error': [] - } + present = [] + mirrored = [] + error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: - add_single_spec(spec, mirror_root, categories, **kwargs) - - return categories['present'], categories['mirrored'], categories['error'] - - -def add_single_spec(spec, mirror_root, categories, **kwargs): - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - spec_exists_in_mirror = True - try: - with spec.package.stage: - # fetcher = stage.fetcher - # fetcher.fetch() - # ... - # fetcher.archive(archive_path) - for ii, stage in enumerate(spec.package.stage): + pkg = spec.package + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + try: + for ii, stage in enumerate(pkg.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -195,7 +184,7 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - spec_exists_in_mirror = False + everything_already_exists = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -206,16 +195,20 @@ def add_single_spec(spec, mirror_root, categories, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if spec_exists_in_mirror: - categories['present'].append(spec) - else: - categories['mirrored'].append(spec) - except Exception as e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - categories['error'].append(spec) + if everything_already_exists: + present.append(spec) + else: + mirrored.append(spec) + except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + error.append(spec) + finally: + pkg.stage.destroy() + + return (present, mirrored, error) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 106c546d5ce..9f1825ca211 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -502,6 +503,7 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher + @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -733,7 +735,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -848,103 +850,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + + + def real_work(): + try: + tty.msg("Building %s." % self.name) + + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + + # Set up process's build environment before running install. + if fake: + self.do_fake_install() else: - self.do_stage() + # Do the real install in the source directory. + self.stage.chdir_to_source() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - def real_work(): - try: - tty.msg("Building %s." % self.name) + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set up process's build environment before running install. - if fake: - self.do_fake_install() - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + tty.msg("Successfully installed %s." % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError, e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + except: + # other exceptions just clean up and raise. + cleanup() + raise - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set parallelism before starting build. + self.make_jobs = make_jobs - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Do the build. + spack.build_environment.fork(self, real_work) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) - - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) - - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) - - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1148,6 +1149,13 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + if os.path.exists(self.stage.path): + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1184,7 +1192,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError as e: + except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 985043c1035..f217450d42d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,26 +42,33 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """A Stage object manages a directory where some source code is + downloaded and built before being installed. It handles + fetching the source code, either as an archive to be expanded + or by checking it out of a repository. A stage's lifecycle + looks like this: - ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. - ``` + Stage() + Constructor creates the stage directory. + fetch() + Fetch a source archive into the stage. + expand_archive() + Expand the source archive. + + Build and install the archive. This is handled by the Package class. + destroy() + Remove the stage once the package has been installed. - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages + in a tmp directory. Otherwise, stages are created directly in + spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can + persist between runs of spack, e.g. if you fetched a tarball but + didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -89,46 +96,21 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - # Try to construct here a temporary name for the stage directory - # If this is a named stage, then construct a named path. - self.path = join_path(spack.stage_path, self.name) - # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.path = None + self._setup() - def __enter__(self): - """ - Entering a stage context will create the stage directory - - Returns: - self - """ - self.create() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """ - Exiting from a stage context will delete the stage directory unless: - - it was explicitly requested not to do so - - an exception has been raised - - Args: - exc_type: exception type - exc_val: exception value - exc_tb: exception traceback - - Returns: - Boolean - """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit - - if self.delete_on_exit: - self.destroy() + def _cleanup_dead_links(self): + """Remove any dead links in the stage directory.""" + for file in os.listdir(spack.stage_path): + path = join_path(spack.stage_path, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(path): + os.unlink(path) def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -166,6 +148,54 @@ def _need_to_create_path(self): return False + def _setup(self): + """Creates the stage directory. + If spack.use_tmp_stage is False, the stage directory is created + directly under spack.stage_path. + + If spack.use_tmp_stage is True, this will attempt to create a + stage in a temporary directory and link it into spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs to + create a stage. If there is no valid location in tmp_dirs, fall + back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + self._cleanup_dead_links() + + # If this is a named stage, then construct a named path. + if self.name is not None: + self.path = join_path(spack.stage_path, self.name) + + # If this is a temporary stage, them make the temp directory + tmp_dir = None + if self.tmp_root: + if self.name is None: + # Unnamed tmp root. Link the path in + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + self.name = os.path.basename(tmp_dir) + self.path = join_path(spack.stage_path, self.name) + if self._need_to_create_path(): + os.symlink(tmp_dir, self.path) + + else: + if self._need_to_create_path(): + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + + # if we're not using a tmp dir, create the stage directly in the + # stage dir, rather than linking to it. + else: + if self.name is None: + self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) + self.name = os.path.basename(self.path) + else: + if self._need_to_create_path(): + mkdirp(self.path) + + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -246,7 +276,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError as e: + except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -298,34 +328,8 @@ def restage(self): """ self.fetcher.reset() - def create(self): - """ - Creates the stage directory - - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. - - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path - if self._need_to_create_path(): - if self.tmp_root: - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - else: - mkdirp(self.path) - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - def destroy(self): - """ - Removes this stage directory - """ + """Remove this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -385,15 +389,6 @@ def source_path(self): def path(self): return self[0].path - def __enter__(self): - for item in self: - item.__enter__() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for item in reversed(self): - item.__exit__(exc_type, exc_val, exc_tb) - def chdir_to_source(self): return self[0].chdir_to_source() @@ -444,6 +439,19 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) +def remove_linked_tree(path): + """Removes a directory and its contents. If the directory is a symlink, + follows the link and reamoves the real directory before removing the + link. + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) + + def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 794344fb6ae..7f2938aec5a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,6 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d6209..d8be5a855b8 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os +import unittest import shutil +import os from tempfile import mkdtemp - +from ordereddict_backport import OrderedDict import spack import spack.config -from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 2440d120e5d..a4e8565b62b 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,15 +23,20 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest import shutil import tempfile -import unittest from llnl.util.filesystem import * + from spack.cmd.create import ConfigureGuesser from spack.stage import Stage -from spack.test.mock_packages_test import * + +from spack.fetch_strategy import URLFetchStrategy +from spack.directory_layout import YamlDirectoryLayout from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockArchive class InstallTest(unittest.TestCase): @@ -47,6 +52,8 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) + if self.stage: + self.stage.destroy() os.chdir(self.orig_dir) @@ -57,12 +64,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - with Stage(url) as stage: - stage.fetch() + self.stage = Stage(url) + self.stage.fetch() - guesser = ConfigureGuesser() - guesser(stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(self.stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9a57e1f03ea..0205f4b8ce2 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,18 +26,19 @@ These tests check the database is functioning properly, both in memory and in its file """ -import multiprocessing -import shutil import tempfile +import shutil +import multiprocessing + +from llnl.util.lock import * +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path -from llnl.util.lock import * -from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * +from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index d814572d4a0..925cb648edd 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,17 +25,20 @@ """\ This test verifies that the Spack directory layout works properly. """ -import os -import shutil +import unittest import tempfile +import shutil +import os + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout -from spack.repository import RepoPath from spack.spec import Spec +from spack.repository import RepoPath +from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * + # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 35780441166..d84433176a8 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,12 +23,19 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import unittest +import shutil +import tempfile + +from llnl.util.filesystem import * import spack -from llnl.util.filesystem import * +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which + from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo -from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -45,15 +52,19 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -65,27 +76,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index b8a0c1ec460..bbcb64e4c1a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,12 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import spack +import unittest -from spack.version import ver -from spack.test.mock_repo import MockHgRepo from llnl.util.filesystem import * + +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which from spack.test.mock_packages_test import * +from spack.test.mock_repo import MockHgRepo class HgFetchTest(MockPackagesTest): @@ -45,10 +49,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def try_fetch(self, rev, test_file, args): """Tries to: @@ -61,27 +68,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8297893f012..8863d13c426 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,13 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import unittest import shutil import tempfile -import spack from llnl.util.filesystem import * -from spack.directory_layout import YamlDirectoryLayout + +import spack +from spack.stage import Stage from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from spack.directory_layout import YamlDirectoryLayout +from spack.util.executable import which from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index ee37e765c7b..886b7ef4c5d 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,6 +24,8 @@ ############################################################################## import os import unittest +import shutil +import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -36,7 +38,6 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') - self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -50,8 +51,10 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) + def tearDown(self): - self.stage.destroy() + if self.stage: + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 3b11d18da49..bc68df01dbe 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,13 +25,15 @@ """ These tests ensure that our lock works correctly. """ -import shutil -import tempfile import unittest +import os +import tempfile +import shutil from multiprocessing import Process -from llnl.util.filesystem import join_path, touch from llnl.util.lock import * +from llnl.util.filesystem import join_path, touch + from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index a2606acf19b..d568a28d443 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import shutil -import tempfile import unittest +import tempfile +import shutil from llnl.util.filesystem import * -from spack.build_environment import MakeExecutable from spack.util.environment import path_put_first +from spack.build_environment import MakeExecutable class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index e707adfe9d5..f83cc8090c9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,10 +23,11 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +from filecmp import dircmp + import spack import spack.mirror - -from filecmp import dircmp +from spack.util.compression import decompressor_for from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -73,14 +74,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - with Stage('spack-mirror-test') as stage: - mirror_root = join_path(stage.path, 'test-mirror') - - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) + stage = Stage('spack-mirror-test') + mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -96,28 +97,38 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - saved_checksum_setting = spack.do_checksum - with pkg.stage: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, pkg.stage.source_path) - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - spack.do_checksum = saved_checksum_setting + pkg._stage = None + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + + dcmp = dircmp(original_path, pkg.stage.source_path) + + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() + finally: + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0b8867b61e2..e9f1f95df55 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,15 +22,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import os import shutil -import tempfile import unittest +import tempfile +from ordereddict_backport import OrderedDict + +from llnl.util.filesystem import mkdirp import spack import spack.config -from llnl.util.filesystem import mkdirp -from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index a8bdfb55710..ed94023b0eb 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,9 +26,13 @@ import shutil from llnl.util.filesystem import * + +import spack +from spack.version import ver from spack.stage import Stage from spack.util.executable import which + # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 2d4b8cd584a..7bf4ff0a0a5 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,11 +25,14 @@ """ Test for multi_method dispatch. """ +import unittest import spack from spack.multimethod import * -from spack.test.mock_packages_test import * from spack.version import * +from spack.spec import Spec +from spack.multimethod import when +from spack.test.mock_packages_test import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 2023ba6d96f..647976df210 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,7 +23,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index 55f35ea4c96..ebd72819992 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest -from spack.spec import Spec +import spack +from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index f0b5e05f3b5..83984dc5f65 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,12 +22,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest + +from llnl.util.filesystem import join_path import spack -from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.test.mock_packages_test import * from spack.util.naming import mod_to_class +from spack.test.mock_packages_test import * class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 42949753044..d74d3b9b7d7 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,11 +28,12 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ +import unittest import os import re -import unittest import llnl.util.tty as tty + import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 5e6162b6e66..632f777cdeb 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,6 +31,8 @@ import spack import spack.package +from llnl.util.lang import list_modules + from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8c33d1ff6e7..44a09cbd7fb 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 6e08e30e135..1daaa4be8fc 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,10 +23,9 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest - import spack.spec -from spack.parse import Token from spack.spec import * +from spack.parse import Token # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index dbcf89d8645..c1b2a2a5730 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,13 +25,15 @@ """\ Test that the Stage class works correctly. """ -import os -import shutil import unittest +import shutil +import os +import getpass from contextlib import * -import spack from llnl.util.filesystem import * + +import spack from spack.stage import Stage from spack.util.executable import which @@ -190,90 +192,116 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url, name=stage_name) as stage: - self.check_setup(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + self.check_setup(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - with Stage(archive_url) as stage: - self.check_setup(stage, None) + stage = Stage(archive_url) + self.check_setup(stage, None) + + stage.destroy() self.check_destroy(stage, None) def test_chdir(self): - with Stage(archive_url, name=stage_name) as stage: - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_fetch(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - self.check_expand_archive(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + self.check_expand_archive(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_expand_archive(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) + + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) + + stage.destroy() self.check_destroy(stage, stage_name) def test_restage(self): - with Stage(archive_url, name=stage_name) as stage: - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + stage = Stage(archive_url, name=stage_name) - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - self.assertTrue('foobar' in os.listdir(stage.source_path)) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) + self.assertTrue('foobar' in os.listdir(stage.source_path)) + + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) + + stage.destroy() self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 1ee4ee700e7..454a7f1d1ff 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,13 +24,19 @@ ############################################################################## import os import re -import spack +import unittest +import shutil +import tempfile -from spack.test.mock_repo import svn, MockSvnRepo -from spack.version import ver -from spack.test.mock_packages_test import * from llnl.util.filesystem import * +import spack +from spack.version import ver +from spack.stage import Stage +from spack.util.executable import which +from spack.test.mock_packages_test import * +from spack.test.mock_repo import svn, MockSvnRepo + class SvnFetchTest(MockPackagesTest): """Tests fetching from a dummy git repository.""" @@ -45,10 +51,13 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) + def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() + self.pkg.do_clean() + def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -61,6 +70,7 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) + def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -72,27 +82,26 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - with self.pkg.stage: - self.pkg.do_stage() - self.assert_rev(rev) + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 4163ab95dd6..e0b9618e0cb 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os - from nose.plugins import Plugin +import os + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 18615b7efe8..ccc409dd602 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,11 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import itertools import unittest +import itertools import spack - test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 068a335b491..87adf894017 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,7 +25,10 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ +import spack import spack.url as url +from spack.spec import Spec +from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index 561d4658a10..efde7c0c73b 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest - import spack.url as url +from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index 2be38af0d34..aec8baf4eaa 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,6 +27,7 @@ """ import unittest +import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 2732006eb3d..108450e098e 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,7 +28,6 @@ where it makes sense. """ import unittest - from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index b930c022f26..5a357b8e696 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,7 +26,6 @@ Test Spack's custom YAML format. """ import unittest - import spack.util.spack_yaml as syaml test_file = """\ From 311d71ec86bc7e245520a816cee4053c31a8a086 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 3 Mar 2016 16:50:51 -0600 Subject: [PATCH 183/339] Modifications to get PGI working --- lib/spack/env/cc | 6 +++--- lib/spack/env/pgi/case-insensitive/pgCC | 1 - lib/spack/env/pgi/{pgf77 => pgc++} | 0 lib/spack/env/pgi/{pgf90 => pgfortran} | 0 lib/spack/spack/compilers/pgi.py | 18 +++++++++--------- 5 files changed, 12 insertions(+), 13 deletions(-) delete mode 120000 lib/spack/env/pgi/case-insensitive/pgCC rename lib/spack/env/pgi/{pgf77 => pgc++} (100%) rename lib/spack/env/pgi/{pgf90 => pgfortran} (100%) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index a323c481243..b8b6c86e010 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -90,15 +90,15 @@ case "$command" in command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|icpc|pgCC|xlc++) + c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) + f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf|nagfor) + f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC deleted file mode 120000 index e2deb67f3b6..00000000000 --- a/lib/spack/env/pgi/case-insensitive/pgCC +++ /dev/null @@ -1 +0,0 @@ -../../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/pgi/pgc++ similarity index 100% rename from lib/spack/env/pgi/pgf77 rename to lib/spack/env/pgi/pgc++ diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgfortran similarity index 100% rename from lib/spack/env/pgi/pgf90 rename to lib/spack/env/pgi/pgfortran diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 9ac74cfbdb5..c6a1078bd9f 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -29,28 +29,28 @@ class Pgi(Compiler): cc_names = ['pgcc'] # Subclasses use possible names of C++ compiler - cxx_names = ['pgCC'] + cxx_names = ['pgc++', 'pgCC'] # Subclasses use possible names of Fortran 77 compiler - f77_names = ['pgf77'] + f77_names = ['pgfortran', 'pgf77'] # Subclasses use possible names of Fortran 90 compiler - fc_names = ['pgf95', 'pgf90'] + fc_names = ['pgfortran', 'pgf95', 'pgf90'] # Named wrapper links within spack.build_env_path link_paths = { 'cc' : 'pgi/pgcc', - 'cxx' : 'pgi/case-insensitive/pgCC', - 'f77' : 'pgi/pgf77', - 'fc' : 'pgi/pgf90' } + 'cxx' : 'pgi/pgc++', + 'f77' : 'pgi/pgfortran', + 'fc' : 'pgi/pgfortran' } @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. Output looks like this:: - pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64 - Copyright 1989-2000, The Portland Group, Inc. All Rights Reserved. - Copyright 2000-2010, STMicroelectronics, Inc. All Rights Reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') From 202fbc681265bd26739e0af3c37bb09a80b7e564 Mon Sep 17 00:00:00 2001 From: citibeth Date: Thu, 3 Mar 2016 23:40:29 -0500 Subject: [PATCH 184/339] Rmoved periods. Should help use of cut'n'paste from console windows. --- lib/spack/spack/cmd/checksum.py | 6 +++--- lib/spack/spack/cmd/compiler.py | 6 +++--- lib/spack/spack/cmd/create.py | 10 +++++----- lib/spack/spack/cmd/mirror.py | 4 ++-- lib/spack/spack/cmd/module.py | 6 +++--- lib/spack/spack/cmd/repo.py | 12 ++++++------ lib/spack/spack/config.py | 6 +++--- lib/spack/spack/database.py | 2 +- lib/spack/spack/directives.py | 4 ++-- lib/spack/spack/directory_layout.py | 4 ++-- lib/spack/spack/fetch_strategy.py | 16 ++++++++-------- lib/spack/spack/mirror.py | 4 ++-- lib/spack/spack/package.py | 26 +++++++++++++------------- lib/spack/spack/repository.py | 6 +++--- lib/spack/spack/stage.py | 4 ++-- 15 files changed, 58 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c4519932335..966ff9a5e96 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -100,11 +100,11 @@ def checksum(parser, args): else: versions = pkg.fetch_remote_versions() if not versions: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) sorted_versions = sorted(versions, reverse=True) - tty.msg("Found %s versions of %s." % (len(versions), pkg.name), + tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) print @@ -121,7 +121,7 @@ def checksum(parser, args): keep_stage=args.keep_stage) if not version_hashes: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 75b51f6b495..3e58e821847 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -96,7 +96,7 @@ def compiler_remove(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.die("No compilers match spec %s." % cspec) + tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: tty.error("Multiple compilers match spec %s. Choose one:" % cspec) colify(reversed(sorted([c.spec for c in compilers])), indent=4) @@ -105,7 +105,7 @@ def compiler_remove(args): for compiler in compilers: spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) - tty.msg("Removed compiler %s." % compiler.spec) + tty.msg("Removed compiler %s" % compiler.spec) def compiler_info(args): @@ -114,7 +114,7 @@ def compiler_info(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.error("No compilers match spec %s." % cspec) + tty.error("No compilers match spec %s" % cspec) else: for c in compilers: print str(c.spec) + ":" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 6809209046e..4564143f834 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -156,7 +156,7 @@ def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: - tty.die("Couldn't guess a version string from %s." % url) + tty.die("Couldn't guess a version string from %s" % url) # Try to guess a name. If it doesn't work, allow the user to override. if args.alternate_name: @@ -189,7 +189,7 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s." + tty.die("Can't create package with namespace %s in repo with namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) @@ -252,7 +252,7 @@ def create(parser, args): name = spec.name # factors out namespace, if any repo = find_repository(spec, args) - tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("This looks like a URL for %s version %s" % (name, version)) tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) @@ -266,7 +266,7 @@ def create(parser, args): keep_stage=args.keep_stage) if not ver_hash_tuples: - tty.die("Could not fetch any tarballs for %s." % name) + tty.die("Could not fetch any tarballs for %s" % name) # Prepend 'py-' to python package names, by convention. if guesser.build_system == 'python': @@ -291,4 +291,4 @@ def create(parser, args): # If everything checks out, go ahead and edit. spack.editor(pkg_path) - tty.msg("Created package %s." % pkg_path) + tty.msg("Created package %s" % pkg_path) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 8e9438c1a37..fcd15a6a904 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -126,7 +126,7 @@ def mirror_remove(args): old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % (name, old_value)) + tty.msg("Removed mirror %s with url %s" % (name, old_value)) def mirror_list(args): @@ -203,7 +203,7 @@ def mirror_create(args): verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, directory), + "Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index a5a9570eb52..1d6867c1d96 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -58,7 +58,7 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s." % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: @@ -78,7 +78,7 @@ def module_find(mtype, spec_array): mt = module_types[mtype] mod = mt(specs[0]) if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s." % (mtype, spec)) + tty.die("No %s module is installed for %s" % (mtype, spec)) print mod.use_name @@ -94,7 +94,7 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s." % spec) + tty.debug(" Writing file for %s" % spec) cls(spec).write() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index c2e352786d1..87c782000f9 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -89,11 +89,11 @@ def repo_add(args): # check if the path exists if not os.path.exists(canon_path): - tty.die("No such file or directory: '%s'." % path) + tty.die("No such file or directory: %s" % path) # Make sure the path is a directory. if not os.path.isdir(canon_path): - tty.die("Not a Spack repository: '%s'." % path) + tty.die("Not a Spack repository: %s" % path) # Make sure it's actually a spack repository by constructing it. repo = Repo(canon_path) @@ -103,7 +103,7 @@ def repo_add(args): if not repos: repos = [] if repo.root in repos or path in repos: - tty.die("Repository is already registered with Spack: '%s'" % path) + tty.die("Repository is already registered with Spack: %s" % path) repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) @@ -122,7 +122,7 @@ def repo_remove(args): if canon_path == repo_canon_path: repos.remove(repo_path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % repo_path) + tty.msg("Removed repository %s" % repo_path) return # If it is a namespace, remove corresponding repo @@ -132,13 +132,13 @@ def repo_remove(args): if repo.namespace == path_or_namespace: repos.remove(path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s' with namespace %s." + tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace)) return except RepoError as e: continue - tty.die("No repository with path or namespace: '%s'" + tty.die("No repository with path or namespace: %s" % path_or_namespace) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6fecde99801..576a5afa2ee 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -205,7 +205,7 @@ def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s." + raise ValueError("Invalid config section: '%s'. Options are %s" % (section, section_schemas)) @@ -335,7 +335,7 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s." + raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, config_scopes.keys())) @@ -350,7 +350,7 @@ def _read_config_file(filename, schema): "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - raise ConfigFileError("Config file is not readable: %s." % filename) + raise ConfigFileError("Config file is not readable: %s" % filename) try: tty.debug("Reading config file %s" % filename) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9cbe7de44a3..089d29325e6 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -330,7 +330,7 @@ def _check_ref_counts(self): found = rec.ref_count if not expected == found: raise AssertionError( - "Invalid ref_count: %s: %d (expected %d), in DB %s." + "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path)) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index c8542f55f02..61cd3030124 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -125,7 +125,7 @@ def __init__(self, dicts=None): dicts = (dicts,) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s." + "dicts arg must be list, tuple, or string. Found %s" % type(dicts)) self.dicts = dicts @@ -317,5 +317,5 @@ class CircularReferenceError(DirectiveError): def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, - "Package '%s' cannot pass itself to %s." % (package, directive)) + "Package '%s' cannot pass itself to %s" % (package, directive)) self.package = package diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 29d87b65b33..08c23627f46 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -335,7 +335,7 @@ def _extension_map(self, spec): if not dag_hash in by_hash: raise InvalidExtensionSpecError( - "Spec %s not found in %s." % (dag_hash, prefix)) + "Spec %s not found in %s" % (dag_hash, prefix)) ext_spec = by_hash[dag_hash] if not prefix == ext_spec.prefix: @@ -450,7 +450,7 @@ class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec, conflict): super(ExtensionConflictError, self).__init__( - "%s cannot be installed in %s because it conflicts with %s."% ( + "%s cannot be installed in %s because it conflicts with %s"% ( ext_spec.short_spec, spec.short_spec, conflict.short_spec)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 83a2dbb59c5..ec17cb97f14 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -153,7 +153,7 @@ def fetch(self): self.stage.chdir() if self.archive_file: - tty.msg("Already downloaded %s." % self.archive_file) + tty.msg("Already downloaded %s" % self.archive_file) return tty.msg("Trying to fetch from %s" % self.url) @@ -275,8 +275,8 @@ def check(self): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) + "%s checksum failed for %s" % (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -312,7 +312,7 @@ def __init__(self, name, *rev_types, **kwargs): # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( + "Supply only one of %s to fetch with %s" % ( comma_or(rev_types), name)) # Set attributes for each rev type. @@ -321,7 +321,7 @@ def __init__(self, name, *rev_types, **kwargs): @_needs_stage def check(self): - tty.msg("No checksum needed when fetching with %s." % self.name) + tty.msg("No checksum needed when fetching with %s" % self.name) @_needs_stage def expand(self): @@ -395,7 +395,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] @@ -505,7 +505,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return tty.msg("Trying to check out svn repository: %s" % self.url) @@ -584,7 +584,7 @@ def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fa29e208036..58e31c2c7b5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -73,7 +73,7 @@ def get_matching_versions(specs, **kwargs): # Skip any package that has no known versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s." % pkg.name) + tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue num_versions = kwargs.get('num_versions', 0) @@ -203,7 +203,7 @@ def create(path, specs, **kwargs): if spack.debug: sys.excepthook(*sys.exc_info()) else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) error.append(spec) finally: pkg.stage.destroy() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9f1825ca211..fb96f61de9e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -688,7 +688,7 @@ def do_fetch(self, mirror_only=False): if not ignore_checksum: raise FetchError( - "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) + "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) self.stage.fetch(mirror_only) @@ -722,7 +722,7 @@ def do_patch(self): # If there are no patches, note it. if not self.patches and not has_patch_fun: - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Construct paths to special files in the archive dir used to @@ -745,7 +745,7 @@ def do_patch(self): tty.msg("Already patched %s" % self.name) return elif os.path.isfile(no_patches_file): - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Apply all the patches for specs that match this one @@ -766,10 +766,10 @@ def do_patch(self): if has_patch_fun: try: self.patch() - tty.msg("Ran patch() for %s." % self.name) + tty.msg("Ran patch() for %s" % self.name) patched = True except: - tty.msg("patch() function failed for %s." % self.name) + tty.msg("patch() function failed for %s" % self.name) touch(bad_file) raise @@ -838,7 +838,7 @@ def do_install(self, raise ValueError("Can only install concrete packages.") if os.path.exists(self.prefix): - tty.msg("%s is already installed in %s." % (self.name, self.prefix)) + tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return tty.msg("Installing %s" % self.name) @@ -874,7 +874,7 @@ def cleanup(): def real_work(): try: - tty.msg("Building %s." % self.name) + tty.msg("Building %s" % self.name) # Run the pre-install hook in the child process after # the directory is created. @@ -918,8 +918,8 @@ def real_work(): self._total_time = time.time() - start_time build_time = self._total_time - self._fetch_time - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s" % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) @@ -1025,7 +1025,7 @@ def do_uninstall(self, force=False): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() spack.installed_db.remove(self.spec) - tty.msg("Successfully uninstalled %s." % self.spec.short_spec) + tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) @@ -1072,7 +1072,7 @@ def do_activate(self, force=False): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s." + tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1124,7 +1124,7 @@ def do_deactivate(self, **kwargs): if self.activated: spack.install_layout.remove_extension(self.extendee_spec, self.spec) - tty.msg("Deactivated extension %s for %s." + tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1320,7 +1320,7 @@ class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" def __init__(self, version): super(PackageVersionError, self).__init__( - "Cannot determine a URL automatically for version %s." % version, + "Cannot determine a URL automatically for version %s" % version, "Please provide a url for this version in the package.py file.") diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 8d06fefe7f0..3c3ba08bcc2 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -156,7 +156,7 @@ def _add(self, repo): if repo.namespace in self.by_namespace: raise DuplicateRepoError( - "Package repos '%s' and '%s' both provide namespace %s." + "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes @@ -545,7 +545,7 @@ def get(self, spec, new=False): raise UnknownPackageError(spec.name) if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError("Repository %s does not contain package %s." + raise UnknownPackageError("Repository %s does not contain package %s" % (self.namespace, spec.fullname)) key = hash(spec) @@ -825,7 +825,7 @@ class UnknownPackageError(PackageLoadError): def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in repository %s." % (name, repo) + msg = "Package %s not found in repository %s" % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index f217450d42d..5591cb9ba5a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -306,9 +306,9 @@ def expand_archive(self): archive_dir = self.source_path if not archive_dir: self.fetcher.expand() - tty.msg("Created stage in %s." % self.path) + tty.msg("Created stage in %s" % self.path) else: - tty.msg("Already staged %s in %s." % (self.name, self.path)) + tty.msg("Already staged %s in %s" % (self.name, self.path)) def chdir_to_source(self): """Changes directory to the expanded archive directory. From 0eb6ef2cd031d1668e1891425bdc6234df0594df Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 4 Mar 2016 11:09:40 -0500 Subject: [PATCH 185/339] 1. Removed fake MPI dependency from netcdf-cxx4 and netcdf-fortran. 2. Removed Fortran bootstrap variant from netcdf. Users who need NetCDF Fortran interface should install netcdf-fortran. 3. Added result of ./configure --help on the netcdf-fortran. Verified the package has no additional options that should be exposed through Spack. --- .../builtin/packages/netcdf-cxx4/package.py | 6 - .../packages/netcdf-fortran/package.py | 156 +++++++++++++++++- .../repos/builtin/packages/netcdf/package.py | 12 +- 3 files changed, 154 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index 8d51a106794..fb4c2886cd8 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -7,12 +7,6 @@ class NetcdfCxx4(Package): version('4.2', 'd019853802092cf686254aaba165fc81') - - variant('mpi', default=True, description='Enables MPI parallelism') - - # netcdf-cxx4 doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 9e4aee95fb8..4e0b14d0121 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -10,12 +10,162 @@ class NetcdfFortran(Package): variant('mpi', default=True, description='Enables MPI parallelism') - # netcdf-fortran doesn't really depend (directly) on MPI. However... this - # depndency ensures taht the right version of MPI is selected (eg: ^openmpi) - depends_on('mpi', when='+mpi') depends_on('netcdf') def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") + + + + +# netcdf-fortran configure parameters are below +# --------------------------------------------- +# +# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. +# +# Usage: ./configure [OPTION]... [VAR=VALUE]... +# +# To assign environment variables (e.g., CC, CFLAGS...), specify them as +# VAR=VALUE. See below for descriptions of some of the useful variables. +# +# Defaults for the options are specified in brackets. +# +# Configuration: +# -h, --help display this help and exit +# --help=short display options specific to this package +# --help=recursive display the short help of all the included packages +# -V, --version display version information and exit +# -q, --quiet, --silent do not print `checking ...' messages +# --cache-file=FILE cache test results in FILE [disabled] +# -C, --config-cache alias for `--cache-file=config.cache' +# -n, --no-create do not create output files +# --srcdir=DIR find the sources in DIR [configure dir or `..'] +# +# Installation directories: +# --prefix=PREFIX install architecture-independent files in PREFIX +# [/usr/local] +# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX +# [PREFIX] +# +# By default, `make install' will install all the files in +# `/usr/local/bin', `/usr/local/lib' etc. You can specify +# an installation prefix other than `/usr/local' using `--prefix', +# for instance `--prefix=$HOME'. +# +# For better control, use the options below. +# +# Fine tuning of the installation directories: +# --bindir=DIR user executables [EPREFIX/bin] +# --sbindir=DIR system admin executables [EPREFIX/sbin] +# --libexecdir=DIR program executables [EPREFIX/libexec] +# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] +# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] +# --localstatedir=DIR modifiable single-machine data [PREFIX/var] +# --libdir=DIR object code libraries [EPREFIX/lib] +# --includedir=DIR C header files [PREFIX/include] +# --oldincludedir=DIR C header files for non-gcc [/usr/include] +# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] +# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] +# --infodir=DIR info documentation [DATAROOTDIR/info] +# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] +# --mandir=DIR man documentation [DATAROOTDIR/man] +# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] +# --htmldir=DIR html documentation [DOCDIR] +# --dvidir=DIR dvi documentation [DOCDIR] +# --pdfdir=DIR pdf documentation [DOCDIR] +# --psdir=DIR ps documentation [DOCDIR] +# +# Program names: +# --program-prefix=PREFIX prepend PREFIX to installed program names +# --program-suffix=SUFFIX append SUFFIX to installed program names +# --program-transform-name=PROGRAM run sed PROGRAM on installed program names +# +# System types: +# --build=BUILD configure for building on BUILD [guessed] +# --host=HOST cross-compile to build programs to run on HOST [BUILD] +# --target=TARGET configure for building compilers for TARGET [HOST] +# +# Optional Features: +# --disable-option-checking ignore unrecognized --enable/--with options +# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) +# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] +# --enable-silent-rules less verbose build output (undo: "make V=1") +# --disable-silent-rules verbose build output (undo: "make V=0") +# --enable-maintainer-mode +# enable make rules and dependencies not useful (and +# sometimes confusing) to the casual installer +# --enable-valgrind-tests build with valgrind-tests (valgrind is required, +# static builds only) +# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is +# not enabled, or built on a system without parallel +# I/O support. +# --enable-extra-tests run some extra tests that may not pass because of +# known issues +# --enable-doxygen Enable generation of documentation with doxygen. +# --enable-dot Use dot (provided by graphviz) to generate charts +# and graphs in the doxygen-based documentation. +# --enable-internal-docs Include documentation of library internals. This is +# of interest only to those developing the netCDF +# library. +# --enable-dependency-tracking +# do not reject slow dependency extractors +# --disable-dependency-tracking +# speeds up one-time build +# --disable-f03-compiler-check +# disable check of ISO_C_BINDING support in Fortran +# compiler +# --disable-f03 suppress netCDF Fortran 2003 native code +# --disable-fortran-type-check +# cause the Fortran type sizes checks to be skipped +# --enable-large-file-tests +# Run tests which create very large data files (~13 GB +# disk space required, but it will be recovered when +# tests are complete). See option --with-temp-large to +# specify temporary directory +# --enable-benchmarks Run benchmarks. This is an experimental feature. +# --enable-shared[=PKGS] build shared libraries [default=yes] +# --enable-static[=PKGS] build static libraries [default=yes] +# --enable-fast-install[=PKGS] +# optimize for fast installation [default=yes] +# --disable-libtool-lock avoid locking (might break parallel builds) +# --disable-largefile omit support for large files +# --enable-extra-example-tests +# Run extra example tests; requires GNU sed. Ignored +# if netCDF-4 is not enabled. +# --enable-dll build a win32 DLL (only works on mingw) +# +# Optional Packages: +# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] +# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) +# --with-temp-large= +# specify directory where large files (i.e. >2 GB) +# will be written, if large files tests are run with +# --enable-large-file-tests +# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use +# both] +# --with-gnu-ld assume the C compiler uses GNU ld [default=no] +# --with-sysroot=DIR Search for dependent libraries within DIR +# (or the compiler's sysroot if not specified). +# +# Some influential environment variables: +# CC C compiler command +# CFLAGS C compiler flags +# LDFLAGS linker flags, e.g. -L if you have libraries in a +# nonstandard directory +# LIBS libraries to pass to the linker, e.g. -l +# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if +# you have headers in a nonstandard directory +# FC Fortran compiler command +# FCFLAGS Fortran compiler flags +# F77 Fortran 77 compiler command +# FFLAGS Fortran 77 compiler flags +# CPP C preprocessor +# +# Use these variables to override the choices made by `configure' or to help +# it to find libraries and programs with nonstandard names/locations. +# +# Report bugs to . +# +# from spack import * diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 41a0d2b6f92..0b112a59ce9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -13,7 +13,6 @@ class Netcdf(Package): version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') variant('mpi', default=True, description='Enables MPI parallelism') - variant('fortran', default=False, description="Download and install NetCDF-Fortran") variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: @@ -66,11 +65,7 @@ def install(self, spec, prefix): # Fortran support # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. - # They can be installed separately, but this bootstrap procedure - # should be able to install both at the same time. - # Note: this is a new experimental feature. - if '+fortran' in spec: - config_args.append("--enable-remote-fortran-bootstrap") + # Use the netcdf-fortran package to install Fortran support. config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) @@ -79,8 +74,3 @@ def install(self, spec, prefix): configure(*config_args) make() make("install") - - # After installing NetCDF-C, install NetCDF-Fortran - if '+fortran' in spec: - make("build-netcdf-fortran") - make("install-netcdf-fortran") From 5db967390450659f794d202bfb5c4b4f11fb964b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 10:51:41 -0600 Subject: [PATCH 186/339] Typo fix in compiler docstring --- lib/spack/spack/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 12c02e0ea2b..d38c0b00b12 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -256,12 +256,12 @@ def find(cls, *path): def __repr__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return self.__str__() def __str__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return "%s(%s)" % ( self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) From b043c4a5b83f36ad4837fc7302f0919a3d759940 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:04:41 -0500 Subject: [PATCH 187/339] Update package.py Removed comments on configure options. --- .../packages/netcdf-fortran/package.py | 153 ------------------ 1 file changed, 153 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 4e0b14d0121..954e7dc3e81 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -16,156 +16,3 @@ def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") - - - - -# netcdf-fortran configure parameters are below -# --------------------------------------------- -# -# `configure' configures netCDF-Fortran 4.4.3 to adapt to many kinds of systems. -# -# Usage: ./configure [OPTION]... [VAR=VALUE]... -# -# To assign environment variables (e.g., CC, CFLAGS...), specify them as -# VAR=VALUE. See below for descriptions of some of the useful variables. -# -# Defaults for the options are specified in brackets. -# -# Configuration: -# -h, --help display this help and exit -# --help=short display options specific to this package -# --help=recursive display the short help of all the included packages -# -V, --version display version information and exit -# -q, --quiet, --silent do not print `checking ...' messages -# --cache-file=FILE cache test results in FILE [disabled] -# -C, --config-cache alias for `--cache-file=config.cache' -# -n, --no-create do not create output files -# --srcdir=DIR find the sources in DIR [configure dir or `..'] -# -# Installation directories: -# --prefix=PREFIX install architecture-independent files in PREFIX -# [/usr/local] -# --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX -# [PREFIX] -# -# By default, `make install' will install all the files in -# `/usr/local/bin', `/usr/local/lib' etc. You can specify -# an installation prefix other than `/usr/local' using `--prefix', -# for instance `--prefix=$HOME'. -# -# For better control, use the options below. -# -# Fine tuning of the installation directories: -# --bindir=DIR user executables [EPREFIX/bin] -# --sbindir=DIR system admin executables [EPREFIX/sbin] -# --libexecdir=DIR program executables [EPREFIX/libexec] -# --sysconfdir=DIR read-only single-machine data [PREFIX/etc] -# --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] -# --localstatedir=DIR modifiable single-machine data [PREFIX/var] -# --libdir=DIR object code libraries [EPREFIX/lib] -# --includedir=DIR C header files [PREFIX/include] -# --oldincludedir=DIR C header files for non-gcc [/usr/include] -# --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] -# --datadir=DIR read-only architecture-independent data [DATAROOTDIR] -# --infodir=DIR info documentation [DATAROOTDIR/info] -# --localedir=DIR locale-dependent data [DATAROOTDIR/locale] -# --mandir=DIR man documentation [DATAROOTDIR/man] -# --docdir=DIR documentation root [DATAROOTDIR/doc/netcdf-fortran] -# --htmldir=DIR html documentation [DOCDIR] -# --dvidir=DIR dvi documentation [DOCDIR] -# --pdfdir=DIR pdf documentation [DOCDIR] -# --psdir=DIR ps documentation [DOCDIR] -# -# Program names: -# --program-prefix=PREFIX prepend PREFIX to installed program names -# --program-suffix=SUFFIX append SUFFIX to installed program names -# --program-transform-name=PROGRAM run sed PROGRAM on installed program names -# -# System types: -# --build=BUILD configure for building on BUILD [guessed] -# --host=HOST cross-compile to build programs to run on HOST [BUILD] -# --target=TARGET configure for building compilers for TARGET [HOST] -# -# Optional Features: -# --disable-option-checking ignore unrecognized --enable/--with options -# --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) -# --enable-FEATURE[=ARG] include FEATURE [ARG=yes] -# --enable-silent-rules less verbose build output (undo: "make V=1") -# --disable-silent-rules verbose build output (undo: "make V=0") -# --enable-maintainer-mode -# enable make rules and dependencies not useful (and -# sometimes confusing) to the casual installer -# --enable-valgrind-tests build with valgrind-tests (valgrind is required, -# static builds only) -# --enable-parallel-tests Run extra parallel IO tests. Ignored if netCDF-4 is -# not enabled, or built on a system without parallel -# I/O support. -# --enable-extra-tests run some extra tests that may not pass because of -# known issues -# --enable-doxygen Enable generation of documentation with doxygen. -# --enable-dot Use dot (provided by graphviz) to generate charts -# and graphs in the doxygen-based documentation. -# --enable-internal-docs Include documentation of library internals. This is -# of interest only to those developing the netCDF -# library. -# --enable-dependency-tracking -# do not reject slow dependency extractors -# --disable-dependency-tracking -# speeds up one-time build -# --disable-f03-compiler-check -# disable check of ISO_C_BINDING support in Fortran -# compiler -# --disable-f03 suppress netCDF Fortran 2003 native code -# --disable-fortran-type-check -# cause the Fortran type sizes checks to be skipped -# --enable-large-file-tests -# Run tests which create very large data files (~13 GB -# disk space required, but it will be recovered when -# tests are complete). See option --with-temp-large to -# specify temporary directory -# --enable-benchmarks Run benchmarks. This is an experimental feature. -# --enable-shared[=PKGS] build shared libraries [default=yes] -# --enable-static[=PKGS] build static libraries [default=yes] -# --enable-fast-install[=PKGS] -# optimize for fast installation [default=yes] -# --disable-libtool-lock avoid locking (might break parallel builds) -# --disable-largefile omit support for large files -# --enable-extra-example-tests -# Run extra example tests; requires GNU sed. Ignored -# if netCDF-4 is not enabled. -# --enable-dll build a win32 DLL (only works on mingw) -# -# Optional Packages: -# --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] -# --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) -# --with-temp-large= -# specify directory where large files (i.e. >2 GB) -# will be written, if large files tests are run with -# --enable-large-file-tests -# --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use -# both] -# --with-gnu-ld assume the C compiler uses GNU ld [default=no] -# --with-sysroot=DIR Search for dependent libraries within DIR -# (or the compiler's sysroot if not specified). -# -# Some influential environment variables: -# CC C compiler command -# CFLAGS C compiler flags -# LDFLAGS linker flags, e.g. -L if you have libraries in a -# nonstandard directory -# LIBS libraries to pass to the linker, e.g. -l -# CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if -# you have headers in a nonstandard directory -# FC Fortran compiler command -# FCFLAGS Fortran compiler flags -# F77 Fortran 77 compiler command -# FFLAGS Fortran 77 compiler flags -# CPP C preprocessor -# -# Use these variables to override the choices made by `configure' or to help -# it to find libraries and programs with nonstandard names/locations. -# -# Report bugs to . -# -# from spack import * From 648d08eb305e04d95ee702b819923f31fd829955 Mon Sep 17 00:00:00 2001 From: Elizabeth Fischer Date: Fri, 4 Mar 2016 12:05:43 -0500 Subject: [PATCH 188/339] Update package.py Updated homepage URL to general NetCDF homepage. --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index fb4c2886cd8..ab717ac6ffb 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -2,7 +2,7 @@ class NetcdfCxx4(Package): """C++ interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" version('4.2', 'd019853802092cf686254aaba165fc81') From a7b918837eab02ebff5af783d06e56ec7f868d9e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 4 Mar 2016 15:22:28 -0600 Subject: [PATCH 189/339] GNU m4 depends on libsigsegv --- .../repos/builtin/packages/libsigsegv/package.py | 15 +++++++++++++++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 var/spack/repos/builtin/packages/libsigsegv/package.py diff --git a/var/spack/repos/builtin/packages/libsigsegv/package.py b/var/spack/repos/builtin/packages/libsigsegv/package.py new file mode 100644 index 00000000000..4b486198ec6 --- /dev/null +++ b/var/spack/repos/builtin/packages/libsigsegv/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Libsigsegv(Package): + """GNU libsigsegv is a library for handling page faults in user mode.""" + homepage = "https://www.gnu.org/software/libsigsegv/" + url = "ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz" + + version('2.10', '7f96fb1f65b3b8cbc1582fb7be774f0f') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-shared') + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 5d76d8866bc..3890663ad12 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,6 +7,8 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + depends_on('libsigsegv') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From f90eaa5f46bfd64b9b0adb4ab5a58d169efe3c69 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 4 Mar 2016 14:53:08 -0800 Subject: [PATCH 190/339] Fixed unmatched function signature for do_fetch in jdk package --- var/spack/repos/builtin/packages/jdk/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index f8f5fc21bdd..cbcc53ac0a7 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -28,7 +28,7 @@ class Jdk(Package): '-H', # specify required License Agreement cookie 'Cookie: oraclelicense=accept-securebackup-cookie'] - def do_fetch(self): + def do_fetch(self, mirror_only=False): # Add our custom curl commandline options tty.msg( "[Jdk] Adding required commandline options to curl " + @@ -39,7 +39,7 @@ def do_fetch(self): spack.curl.add_default_arg(option) # Now perform the actual fetch - super(Jdk, self).do_fetch() + super(Jdk, self).do_fetch(mirror_only) def install(self, spec, prefix): From f663d37da75e4ef0f266c63707e77d29a651e07d Mon Sep 17 00:00:00 2001 From: Mark Miller Date: Fri, 4 Mar 2016 17:08:11 -0800 Subject: [PATCH 191/339] Adding zfp package. --- .../repos/builtin/packages/zfp/package.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 var/spack/repos/builtin/packages/zfp/package.py diff --git a/var/spack/repos/builtin/packages/zfp/package.py b/var/spack/repos/builtin/packages/zfp/package.py new file mode 100644 index 00000000000..620fe9d4565 --- /dev/null +++ b/var/spack/repos/builtin/packages/zfp/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Zfp(Package): + """zfp is an open source C library for compressed floating-point arrays that supports + very high throughput read and write random acces, target error bounds or bit rates. + Although bit-for-bit lossless compression is not always possible, zfp is usually + accurate to within machine epsilon in near-lossless mode, and is often orders of + magnitude more accurate than other lossy compressors. + """ + + homepage = "http://computation.llnl.gov/projects/floating-point-compression" + url = "http://computation.llnl.gov/projects/floating-point-compression/download/zfp-0.5.0.tar.gz" + + version('0.5.0', '2ab29a852e65ad85aae38925c5003654') + + def install(self, spec, prefix): + make("shared") + + # No install provided + mkdirp(prefix.lib) + mkdirp(prefix.include) + install('lib/libzfp.so', prefix.lib) + install('inc/zfp.h', prefix.include) + install('inc/types.h', prefix.include) + install('inc/bitstream.h', prefix.include) + install('inc/system.h', prefix.include) From fc6aa7374a243928af38fd1e583c6c67146951cd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:18:48 -0800 Subject: [PATCH 192/339] Fix #104, #54: issues with overlong shebang in deep directories. This does several things: - Add `sbang`: a script to run scripts with long shebang lines. - Documentation for `sbang` is in `bin/sbang`. - Add an `sbang` hook that filters the `bin` directory after install and modifies any scripts wtih shebangs that are too long to use `sbang` instead. - `sbang` is at the top level, so it should be runnable (not much we can do if spack itself is too deep for shebang) - `sbang`, when used as the interpreter, runs the *second* shebang line it finds in a script. - shoud fix issues with too long shebang paths. --- bin/sbang | 84 +++++++++++++++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 11 +++- lib/spack/spack/hooks/sbang.py | 74 +++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100755 bin/sbang create mode 100644 lib/spack/spack/hooks/sbang.py diff --git a/bin/sbang b/bin/sbang new file mode 100755 index 00000000000..ebfbe2e7a16 --- /dev/null +++ b/bin/sbang @@ -0,0 +1,84 @@ +#!/bin/bash +# +# `sbang`: Run scripts with long shebang lines. +# +# Many operating systems limit the length of shebang lines, making it +# hard to use interpreters that are deep in the directory hierarchy. +# `sbang` can run such scripts, either as a shebang interpreter, or +# directly on the command line. +# +# Usage +# ----------------------------- +# Suppose you have a script, long-shebang.sh, like this: +# +# 1 #!/very/long/path/to/some/interpreter +# 2 +# 3 echo "success!" +# +# Invoking this script will result in an error on some OS's. On +# Linux, you get this: +# +# $ ./long-shebang.sh +# -bash: ./long: /very/long/path/to/some/interp: bad interpreter: +# No such file or directory +# +# On Mac OS X, the system simply assumes the interpreter is the shell +# and tries to run with it, which is likely not what you want. +# +# +# `sbang` on the command line +# ----------------------------- +# You can use `sbang` in two ways. The first is to use it directly, +# from the command line, like this: +# +# $ sbang ./long-shebang.sh +# success! +# +# +# `sbang` as the interpreter +# ----------------------------- +# You can also use `sbang` *as* the interpreter for your script. Put +# `#!/bin/bash /path/to/sbang` on line 1, and move the original +# shebang to line 2 of the script: +# +# 1 #!/bin/bash /path/to/sbang +# 2 #!/long/path/to/real/interpreter with arguments +# 3 +# 4 echo "success!" +# +# $ ./long-shebang.sh +# success! +# +# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other +# operating systems like Mac OS X require the interpreter to be a +# binary, so it's best to use `sbang` as a `bash` argument. +# Obviously, for this to work, `sbang` needs to have a short enough +# path that *it* will run without hitting OS limits. +# +# +# How it works +# ----------------------------- +# `sbang` is a very simple bash script. It looks at the first two +# lines of a script argument and runs the last line starting with +# `#!`, with the script as an argument. It also forwards arguments. +# + +# First argument is the script we want to actually run. +script="$1" + +# Search the first two lines of script for interpreters. +lines=0 +while read line && ((lines < 2)) ; do + if [[ "$line" = '#!'* ]]; then + interpreter="${line#\#!}" + fi + lines=$((lines+1)) +done < "$script" + +# Invoke any interpreter found, or raise an error if none was found. +if [ -n "$interpreter" ]; then + exec $interpreter "$@" +else + echo "error: sbang found no interpreter in $script" + exit 1 +fi diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..10d25bdce8d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'copy_mode', 'unset_executable_mode'] import os import sys @@ -158,6 +159,14 @@ def copy_mode(src, dest): os.chmod(dest, dest_mode) +def unset_executable_mode(path): + mode = os.stat(path).st_mode + mode &= ~stat.S_IXUSR + mode &= ~stat.S_IXGRP + mode &= ~stat.S_IXOTH + os.chmod(path, mode) + + def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py new file mode 100644 index 00000000000..6117c4809dc --- /dev/null +++ b/lib/spack/spack/hooks/sbang.py @@ -0,0 +1,74 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +from llnl.util.filesystem import * +import llnl.util.tty as tty + +import spack +import spack.modules + +# Character limit for shebang line. Using Linux's 127 characters +# here, as it is the shortest I could find on a modern OS. +shebang_limit = 127 + +def shebang_too_long(path): + """Detects whether an file has a shebang line that is too long.""" + with open(path, 'r') as script: + bytes = script.read(2) + if bytes != '#!': + return False + + line = bytes + script.readline() + return len(line) > shebang_limit + + +def filter_shebang(path): + """Adds a second shebang line, using sbang, at the beginning of a file.""" + backup = path + ".shebang.bak" + os.rename(path, backup) + + with open(backup, 'r') as bak_file: + original = bak_file.read() + + with open(path, 'w') as new_file: + new_file.write('#!/bin/bash %s/bin/sbang\n' % spack.spack_root) + new_file.write(original) + + copy_mode(backup, path) + unset_executable_mode(backup) + + tty.warn("Patched overly long shebang in %s" % path) + + +def post_install(pkg): + """This hook edits scripts so that they call /bin/bash + $spack_prefix/bin/sbang instead of something longer than the + shebang limit.""" + for file in os.listdir(pkg.prefix.bin): + path = os.path.join(pkg.prefix.bin, file) + if shebang_too_long(path): + filter_shebang(path) + From bfce2c7508ada675ebe4398395b30ede82b67093 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 04:42:52 -0800 Subject: [PATCH 193/339] Fix bug in hook: ensure bin directory exists before listing. --- lib/spack/spack/hooks/sbang.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 6117c4809dc..3390ecea295 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -67,6 +67,9 @@ def post_install(pkg): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit.""" + if not os.path.isdir(pkg.prefix.bin): + return + for file in os.listdir(pkg.prefix.bin): path = os.path.join(pkg.prefix.bin, file) if shebang_too_long(path): From 3dd630d0a567964acd11bee23ebbc52c7d1c61c5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 14:33:23 -0800 Subject: [PATCH 194/339] Make openssl a variant in libevent. --- var/spack/repos/builtin/packages/libevent/package.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 2a44c49325d..714a155dc00 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -22,12 +22,16 @@ class Libevent(Package): version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') version('2.0.12', '42986228baf95e325778ed328a93e070') - - depends_on('openssl') - + variant('openssl', default=True, description="Build with encryption enabled at the libevent level.") + depends_on('openssl', when='+openssl') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if '+openssl' in spec: + configure_args.append('--enable-openssl') + else: + configure_args.append('--enable-openssl') + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 220b6a9fee0f307d4de1e48b29093812f7dd10ec Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sat, 5 Mar 2016 16:46:32 -0600 Subject: [PATCH 195/339] Make libsigsegv an optional dependency --- var/spack/repos/builtin/packages/m4/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 3890663ad12..9d522dfccf3 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,7 +7,9 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') - depends_on('libsigsegv') + variant('sigsegv', default=True, description="Build the libsigsegv dependency") + + depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 7d847f4dc4f8895d57e492c11cfd7c3fbb794945 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 15:14:21 -0800 Subject: [PATCH 196/339] Minor tweaks to m4 -- be sure to add sigsegv args explicitly. --- var/spack/repos/builtin/packages/m4/package.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index 9d522dfccf3..d6829dbcd4b 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -12,6 +12,12 @@ class M4(Package): depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if 'libsigsegv' in spec: + configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) + else: + configure_args.append('--without-libsigsegv-prefix') + + configure("--prefix=%s" % prefix, *configure_args) make() make("install") From 14d48eba461596b3c5b03cfe269318946406e02c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 18:40:28 -0800 Subject: [PATCH 197/339] Revert c5d9ee8924 for bug fixes. - This reverts commit c5d9ee89246b3d2aeddb756a04588424051d3295. - merged too soon before - reverting and fixing bugs now. Conflicts: lib/spack/spack/mirror.py lib/spack/spack/package.py --- lib/spack/llnl/util/filesystem.py | 35 +++- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/mirror.py | 55 +++--- lib/spack/spack/package.py | 175 +++++++++---------- lib/spack/spack/stage.py | 186 ++++++++++----------- lib/spack/spack/test/concretize.py | 2 - lib/spack/spack/test/config.py | 6 +- lib/spack/spack/test/configure_guess.py | 21 +-- lib/spack/spack/test/database.py | 11 +- lib/spack/spack/test/directory_layout.py | 13 +- lib/spack/spack/test/git_fetch.py | 46 ++--- lib/spack/spack/test/hg_fetch.py | 44 +++-- lib/spack/spack/test/install.py | 9 +- lib/spack/spack/test/link_tree.py | 7 +- lib/spack/spack/test/lock.py | 8 +- lib/spack/spack/test/make_executable.py | 6 +- lib/spack/spack/test/mirror.py | 69 ++++---- lib/spack/spack/test/mock_packages_test.py | 8 +- lib/spack/spack/test/mock_repo.py | 4 - lib/spack/spack/test/multimethod.py | 5 +- lib/spack/spack/test/namespace_trie.py | 1 + lib/spack/spack/test/optional_deps.py | 4 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/python_version.py | 3 +- lib/spack/spack/test/spec_dag.py | 2 - lib/spack/spack/test/spec_semantics.py | 1 - lib/spack/spack/test/spec_syntax.py | 3 +- lib/spack/spack/test/stage.py | 130 ++++++-------- lib/spack/spack/test/svn_fetch.py | 49 +++--- lib/spack/spack/test/tally_plugin.py | 4 +- lib/spack/spack/test/unit_install.py | 3 +- lib/spack/spack/test/url_extrapolate.py | 3 - lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/test/versions.py | 1 + lib/spack/spack/test/yaml.py | 1 + 36 files changed, 422 insertions(+), 504 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index da3cf96050f..366237ef8f3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -240,7 +240,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -344,3 +344,34 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real + directory before removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 6e7179122c1..0c8bd1d5281 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_clean() + package.stage.destroy() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 58e31c2c7b5..fdc4e7967f5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -110,7 +110,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -158,17 +157,29 @@ def create(path, specs, **kwargs): "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) - try: - for ii, stage in enumerate(pkg.stage): + add_single_spec(spec, mirror_root, categories, **kwargs) + + return categories['present'], categories['mirrored'], categories['error'] + + +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version @@ -184,7 +195,7 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: - everything_already_exists = False + spec_exists_in_mirror = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() @@ -195,20 +206,16 @@ def create(path, specs, **kwargs): fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) - error.append(spec) - finally: - pkg.stage.destroy() - - return (present, mirrored, error) + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fb96f61de9e..be45415b759 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,7 +293,6 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -503,7 +502,6 @@ def fetcher(self): self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -735,7 +733,7 @@ def do_patch(self): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -850,102 +848,102 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) - - - def real_work(): - try: - tty.msg("Building %s" % self.name) - - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) - - # Set up process's build environment before running install. - if fake: - self.do_fake_install() + with self.stage: + if not fake: + if not skip_patch: + self.do_patch() else: - # Do the real install in the source directory. - self.stage.chdir_to_source() + self.do_stage() - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - dump_environment(env_path) - self.install(self.spec, self.prefix) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - # Ensure that something was actually installed. - self._sanity_check_install() + def real_work(): + try: + tty.msg("Building %s" % self.name) - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - install(log_path, log_install_path) - install(env_path, env_install_path) + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + # Set up process's build environment before running install. + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # This redirects I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s" - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Ensure that something was actually installed. + self._sanity_check_install() - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + # Move build log into install directory on success + if not fake: + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + install(log_path, log_install_path) + install(env_path, env_install_path) - except: - # other exceptions just clean up and raise. - cleanup() - raise + packages_dir = spack.install_layout.build_packages_path(self.spec) + dump_packages(self.spec, packages_dir) - # Set parallelism before starting build. - self.make_jobs = make_jobs + # On successful install, remove the stage. + if not keep_stage: + self.stage.destroy() - # Do the build. - spack.build_environment.fork(self, real_work) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e + + except: + # other exceptions just clean up and raise. + cleanup() + raise + + # Set parallelism before starting build. + self.make_jobs = make_jobs + + # Do the build. + spack.build_environment.fork(self, real_work) + + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): @@ -1149,13 +1147,6 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - - def do_clean(self): - """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() - - def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) @@ -1192,7 +1183,7 @@ def fetch_remote_versions(self): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5591cb9ba5a..a22982a6d43 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,33 +42,26 @@ class Stage(object): - """A Stage object manages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: + """ + A Stage object is a context manager that handles a directory where some source code is downloaded and built + before being installed. It handles fetching the source code, either as an archive to be expanded or by checking + it out of a repository. A stage's lifecycle looks like this: - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + fetch() # Fetch a source archive into the stage. + expand_archive() # Expand the source archive. + # Build and install the archive. This is handled by the Package class. + ``` - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. + If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. + Otherwise, stages are created directly in spack.stage_path. - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. + There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you + fetched a tarball but didn't finish building it, you won't have to fetch it again. - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for + only one run of spack. """ def __init__(self, url_or_fetch_strategy, **kwargs): @@ -96,21 +89,46 @@ def __init__(self, url_or_fetch_strategy, **kwargs): self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. - self.name = kwargs.get('name') + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) self.mirror_path = kwargs.get('mirror_path') self.tmp_root = find_tmp_root() - self.path = None - self._setup() + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not + self.delete_on_exit = True - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(path): - os.unlink(path) + def __enter__(self): + """ + Entering a stage context will create the stage directory + + Returns: + self + """ + self.create() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + + if self.delete_on_exit: + self.destroy() def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we @@ -148,54 +166,6 @@ def _need_to_create_path(self): return False - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - - # If this is a temporary stage, them make the temp directory - tmp_dir = None - if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -276,7 +246,7 @@ def fetch(self, mirror_only=False): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -328,8 +298,34 @@ def restage(self): """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise + this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location + in tmp_dirs, fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) + def destroy(self): - """Remove this stage directory.""" + """ + Removes this stage directory + """ remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -389,6 +385,15 @@ def source_path(self): def path(self): return self[0].path + def __enter__(self): + for item in self: + item.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for item in reversed(self): + item.__exit__(exc_type, exc_val, exc_tb) + def chdir_to_source(self): return self[0].chdir_to_source() @@ -439,19 +444,6 @@ def ensure_access(file=spack.stage_path): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5a..794344fb6ae 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,8 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b8..0562d2d6209 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62b..2440d120e5d 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): @@ -52,8 +47,6 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +57,12 @@ def check_archive(self, filename, system): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce2..9a57e1f03ea 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,18 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648edd..d814572d4a0 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index d84433176a8..35780441166 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -52,19 +45,15 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -76,26 +65,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index bbcb64e4c1a..b8a0c1ec460 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): @@ -49,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def try_fetch(self, rev, test_file, args): """Tries to: @@ -68,26 +61,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 8863d13c426..8297893f012 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5d..ee37e765c7b 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,6 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +50,8 @@ def setUp(self): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01dbe..3b11d18da49 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index d568a28d443..a2606acf19b 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index f83cc8090c9..e707adfe9d5 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -74,14 +73,14 @@ def set_up_package(self, name, MockRepoClass, url_attr): def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - # register mirror with spack config - mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } - spack.config.update_config('mirrors', mirrors) - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -97,38 +96,28 @@ def check_mirror(self): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Now try to fetch each package. - for name, mock_repo in self.repos.items(): - spec = Spec(name).concretized() - pkg = spec.package + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - pkg._stage = None - saved_checksum_setting = spack.do_checksum - try: - # Stage the archive from the mirror and cd to it. - spack.do_checksum = False - pkg.do_stage(mirror_only=True) - - # Compare the original repo with the expanded archive - original_path = mock_repo.path - if 'svn' in name: - # have to check out the svn repo to compare. - original_path = join_path(mock_repo.path, 'checked_out') - svn('checkout', mock_repo.url, original_path) - - dcmp = dircmp(original_path, pkg.stage.source_path) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - - # and that all original files are present. - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - spack.do_checksum = saved_checksum_setting - pkg.do_clean() - finally: - stage.destroy() + saved_checksum_setting = spack.do_checksum + with pkg.stage: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df55..0b8867b61e2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index ed94023b0eb..a8bdfb55710 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a5..2d4b8cd584a 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 647976df210..2023ba6d96f 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd72819992..55f35ea4c96 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f65..f0b5e05f3b5 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d7..42949753044 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cdeb..5e6162b6e66 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7fb..8c33d1ff6e7 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8fc..6e08e30e135 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a5730..dbcf89d8645 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which @@ -192,116 +190,90 @@ def check_destroy(self, stage, stage_name): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) self.check_destroy(stage, stage_name) def test_restage(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + self.assertTrue('foobar' in os.listdir(stage.source_path)) - self.assertTrue('foobar' in os.listdir(stage.source_path)) - - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) - - stage.destroy() + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 454a7f1d1ff..1ee4ee700e7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * + from spack.test.mock_repo import svn, MockSvnRepo +from spack.version import ver +from spack.test.mock_packages_test import * +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): @@ -51,13 +45,10 @@ def setUp(self): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() self.repo.destroy() - self.pkg.do_clean() - def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" @@ -70,7 +61,6 @@ def get_rev(): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -82,26 +72,27 @@ def try_fetch(self, rev, test_file, args): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index e0b9618e0cb..4163ab95dd6 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd602..18615b7efe8 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf894017..068a335b491 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73b..561d4658a10 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4eaa..2be38af0d34 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098e..2732006eb3d 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e696..b930c022f26 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ From ad103dcafa652a839590f5fce28b2e2ce3b5a56d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 5 Mar 2016 19:55:07 -0800 Subject: [PATCH 198/339] Small refactor: add keep parameter to stage, get rid of stage.destroy call. - package.py uses context manager more effectively. - Stage.__init__ has easier to understand method signature now. - keep can be used to override the default behavior either to keep the stage ALL the time or to delete the stage ALL the time. --- lib/spack/llnl/util/filesystem.py | 5 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/package.py | 108 +++++++++++++++------------- lib/spack/spack/stage.py | 116 ++++++++++++++++++++---------- 4 files changed, 141 insertions(+), 90 deletions(-) diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 366237ef8f3..a92cb0706d2 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -362,8 +362,9 @@ def remove_dead_links(root): def remove_linked_tree(path): """ - Removes a directory and its contents. If the directory is a symlink, follows the link and removes the real - directory before removing the link. + Removes a directory and its contents. If the directory is a + symlink, follows the link and removes the real directory before + removing the link. Args: path: directory to be removed diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 0c8bd1d5281..6e7179122c1 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -43,4 +43,4 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.stage.destroy() + package.do_clean() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index be45415b759..47d259968aa 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -293,6 +293,7 @@ class SomePackage(Package): .. code-block:: python + p.do_clean() # removes the stage directory entirely p.do_restage() # removes the build directory and # re-expands the archive. @@ -455,7 +456,7 @@ def _make_stage(self): # Construct a composite stage on top of the composite FetchStrategy composite_fetcher = self.fetcher composite_stage = StageComposite() - resources = self._get_resources() + resources = self._get_needed_resources() for ii, fetcher in enumerate(composite_fetcher): if ii == 0: # Construct root stage first @@ -484,12 +485,14 @@ def stage(self, stage): def _make_fetcher(self): - # Construct a composite fetcher that always contains at least one element (the root package). In case there - # are resources associated with the package, append their fetcher to the composite. + # Construct a composite fetcher that always contains at least + # one element (the root package). In case there are resources + # associated with the package, append their fetcher to the + # composite. root_fetcher = fs.for_package_version(self, self.version) fetcher = fs.FetchStrategyComposite() # Composite fetcher fetcher.append(root_fetcher) # Root fetcher is always present - resources = self._get_resources() + resources = self._get_needed_resources() for resource in resources: fetcher.append(resource.fetcher) return fetcher @@ -706,6 +709,7 @@ def do_stage(self, mirror_only=False): self.stage.expand_archive() self.stage.chdir_to_source() + def do_patch(self): """Calls do_stage(), then applied patches to the expanded tarball if they haven't been applied already.""" @@ -798,7 +802,7 @@ def do_fake_install(self): mkdirp(self.prefix.man1) - def _get_resources(self): + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): @@ -816,7 +820,7 @@ def _resource_stage(self, resource): def do_install(self, - keep_prefix=False, keep_stage=False, ignore_deps=False, + keep_prefix=False, keep_stage=None, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False): """Called by commands to install a package and its dependencies. @@ -825,7 +829,8 @@ def do_install(self, Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Keep stage on successful build. By default, destroys it. + keep_stage -- Set to True or false to always keep or always delete stage. + By default, stage is destroyed only if there are no exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. @@ -848,32 +853,33 @@ def do_install(self, make_jobs=make_jobs) start_time = time.time() - with self.stage: - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + def cleanup(): + if not keep_prefix: + # If anything goes wrong, remove the install prefix + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed." + + "Manually remove this directory to fix:", + self.prefix, wrap=True) - def real_work(): - try: - tty.msg("Building %s" % self.name) + def real_work(): + try: + tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self) @@ -888,7 +894,7 @@ def real_work(): # Save the build environment in a file before building. env_path = join_path(os.getcwd(), 'spack-build.env') - # This redirects I/O to a build log (and optionally to the terminal) + # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') with log_output(log_file, verbose, sys.stdout.isatty(), True): @@ -908,29 +914,25 @@ def real_work(): packages_dir = spack.install_layout.build_packages_path(self.spec) dump_packages(self.spec, packages_dir) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + except ProcessError as e: + # Annotate with location of build log. + e.build_log = log_path + cleanup() + raise e - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e - - except: - # other exceptions just clean up and raise. - cleanup() - raise + except: + # other exceptions just clean up and raise. + cleanup() + raise # Set parallelism before starting build. self.make_jobs = make_jobs @@ -1147,6 +1149,12 @@ def do_restage(self): """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() + + def do_clean(self): + """Removes the package's build stage and source tarball.""" + self.stage.destroy() + + def format_doc(self, **kwargs): """Wrap doc string at 72 characters and format nicely""" indent = kwargs.get('indent', 0) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a22982a6d43..b117c76aa17 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -42,29 +42,53 @@ class Stage(object): - """ - A Stage object is a context manager that handles a directory where some source code is downloaded and built - before being installed. It handles fetching the source code, either as an archive to be expanded or by checking - it out of a repository. A stage's lifecycle looks like this: + """Manages a temporary stage directory for building. + + A Stage object is a context manager that handles a directory where + some source code is downloaded and built before being installed. + It handles fetching the source code, either as an archive to be + expanded or by checking it out of a repository. A stage's + lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory - fetch() # Fetch a source archive into the stage. - expand_archive() # Expand the source archive. - # Build and install the archive. This is handled by the Package class. + with Stage() as stage: # Context manager creates and destroys the stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) ``` - If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. - Otherwise, stages are created directly in spack.stage_path. + When used as a context manager, the stage is automatically + destroyed if no exception is raised by the context. If an + excpetion is raised, the stage is left in the filesystem and NOT + destroyed, for potential reuse later. - There are two kinds of stages: named and unnamed. Named stages can persist between runs of spack, e.g. if you - fetched a tarball but didn't finish building it, you won't have to fetch it again. + You can also use the stage's create/destroy functions manually, + like this: - Unnamed stages are created using standard mkdtemp mechanisms or similar, and are intended to persist for - only one run of spack. + ``` + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. + ``` + + If spack.use_tmp_stage is True, spack will attempt to create + stages in a tmp directory. Otherwise, stages are created directly + in spack.stage_path. + + There are two kinds of stages: named and unnamed. Named stages + can persist between runs of spack, e.g. if you fetched a tarball + but didn't finish building it, you won't have to fetch it again. + + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, **kwargs): + def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -76,6 +100,18 @@ def __init__(self, url_or_fetch_strategy, **kwargs): and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. + + mirror_path + If provided, Stage will search Spack's mirrors for + this archive at the mirror_path, before using the + default fetch strategy. + + keep + By default, when used as a context manager, the Stage + is cleaned up when everything goes well, and it is + kept intact when an exception is raised. You can + override this behavior by setting keep to True + (always keep) or False (always delete). """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -91,15 +127,19 @@ def __init__(self, url_or_fetch_strategy, **kwargs): # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root - self.name = kwargs.get('name') if 'name' in kwargs else STAGE_PREFIX + next(tempfile._get_candidate_names()) - self.mirror_path = kwargs.get('mirror_path') + self.name = name + if name is None: + self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.mirror_path = mirror_path self.tmp_root = find_tmp_root() # Try to construct here a temporary name for the stage directory # If this is a named stage, then construct a named path. self.path = join_path(spack.stage_path, self.name) + # Flag to decide whether to delete the stage folder on exit or not - self.delete_on_exit = True + self.keep = keep + def __enter__(self): """ @@ -111,6 +151,7 @@ def __enter__(self): self.create() return self + def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -125,11 +166,15 @@ def __exit__(self, exc_type, exc_val, exc_tb): Returns: Boolean """ - self.delete_on_exit = False if exc_type is not None else self.delete_on_exit + if self.keep is None: + # Default: delete when there are no exceptions. + if exc_type is None: self.destroy() - if self.delete_on_exit: + elif not self.keep: + # Overridden. Either always keep or always delete. self.destroy() + def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. @@ -201,7 +246,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" @@ -302,11 +347,14 @@ def create(self): """ Creates the stage directory - If self.tmp_root evaluates to False, the stage directory is created directly under spack.stage_path, otherwise - this will attempt to create a stage in a temporary directory and link it into spack.stage_path. + If self.tmp_root evaluates to False, the stage directory is + created directly under spack.stage_path, otherwise this will + attempt to create a stage in a temporary directory and link it + into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to create a stage. If there is no valid location - in tmp_dirs, fall back to making the stage inside spack.stage_path. + Spack will use the first writable location in spack.tmp_dirs + to create a stage. If there is no valid location in tmp_dirs, + fall back to making the stage inside spack.stage_path. """ # Create the top-level stage directory mkdirp(spack.stage_path) @@ -323,9 +371,7 @@ def create(self): ensure_access(self.path) def destroy(self): - """ - Removes this stage directory - """ + """Removes this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -370,7 +416,7 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and @@ -410,7 +456,7 @@ def chdir(self): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) def chdir_to_source(self): self.chdir() @@ -472,19 +518,15 @@ def find_tmp_root(): class StageError(spack.error.SpackError): - def __init__(self, message, long_message=None): - super(self, StageError).__init__(message, long_message) + """"Superclass for all errors encountered during staging.""" class RestageError(StageError): - def __init__(self, message, long_msg=None): - super(RestageError, self).__init__(message, long_msg) + """"Error encountered during restaging.""" class ChdirError(StageError): - def __init__(self, message, long_msg=None): - super(ChdirError, self).__init__(message, long_msg) - + """Raised when Spack can't change directories.""" # Keep this in namespace for convenience FailedDownloadError = fs.FailedDownloadError From ae87948a232c17982a7987e5423ac77acdc59194 Mon Sep 17 00:00:00 2001 From: citibeth Date: Sat, 5 Mar 2016 23:05:45 -0500 Subject: [PATCH 199/339] Fixed URLs Removed vestigal mpi variant in netcdf-fortran --- var/spack/repos/builtin/packages/netcdf-cxx4/package.py | 1 - var/spack/repos/builtin/packages/netcdf-fortran/package.py | 4 +--- var/spack/repos/builtin/packages/netcdf/package.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py index ab717ac6ffb..b83e964b00a 100644 --- a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -11,6 +11,5 @@ class NetcdfCxx4(Package): def install(self, spec, prefix): configure('--prefix=%s' % prefix) - make() make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py index 954e7dc3e81..e4e33445e54 100644 --- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -3,13 +3,11 @@ class NetcdfFortran(Package): """Fortran interface for NetCDF4""" - homepage = "http://www.unidata.ucar.edu/downloads/netcdf/netcdf-cxx/index.jsp" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') - variant('mpi', default=True, description='Enables MPI parallelism') - depends_on('netcdf') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 0b112a59ce9..227362399a9 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -6,7 +6,7 @@ class Netcdf(Package): data formats that support the creation, access, and sharing of array-oriented scientific data.""" - homepage = "http://www.unidata.ucar.edu/software/netcdf/" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') From e515042a36e7aa21e52943dab1e8b5594f3f0e94 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 01:41:48 -0800 Subject: [PATCH 200/339] Fix stage creation bug, simplify do_install code. --- lib/spack/spack/package.py | 73 +++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 47d259968aa..ce8cce27e23 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -467,6 +467,11 @@ def _make_stage(self): stage = self._make_resource_stage(composite_stage[0], fetcher, resource) # Append the item to the composite composite_stage.append(stage) + + # Create stage on first access. Needed because fetch, stage, + # patch, and install can be called independently of each + # other, so `with self.stage:` in do_install isn't sufficient. + composite_stage.create() return composite_stage @property @@ -846,35 +851,38 @@ def do_install(self, tty.msg("Installing %s" % self.name) + # First, install dependencies recursively. if not ignore_deps: self.do_install_dependencies( keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, - make_jobs=make_jobs) - - start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() - - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) def cleanup(): + """Handles removing install prefix on error.""" if not keep_prefix: - # If anything goes wrong, remove the install prefix self.remove_prefix() else: tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + + "Spack will think this package is installed. " + "Manually remove this directory to fix:", self.prefix, wrap=True) + # Then install the package itself. def real_work(): + """Forked for each build. Has its own process and python + module space set up by build_environment.fork().""" + start_time = time.time() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() + + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.create_install_directory(self.spec) + try: tty.msg("Building %s" % self.name) @@ -884,9 +892,9 @@ def real_work(): # the directory is created. spack.hooks.pre_install(self) - # Set up process's build environment before running install. if fake: self.do_fake_install() + else: # Do the real install in the source directory. self.stage.chdir_to_source() @@ -901,18 +909,17 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + # Ensure that something was actually installed. + self._sanity_check_install() - # Move build log into install directory on success - if not fake: + # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) + install(log_path, log_install_path) install(env_path, env_install_path) - - packages_dir = spack.install_layout.build_packages_path(self.spec) - dump_packages(self.spec, packages_dir) + dump_packages(self.spec, packages_dir) # Stop timer. self._total_time = time.time() - start_time @@ -934,18 +941,18 @@ def real_work(): cleanup() raise - # Set parallelism before starting build. - self.make_jobs = make_jobs + # Set parallelism before starting build. + self.make_jobs = make_jobs - # Do the build. - spack.build_environment.fork(self, real_work) + # Do the build. + spack.build_environment.fork(self, real_work) - # note: PARENT of the build process adds the new package to - # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, self.prefix) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) - # Once everything else is done, run post install hooks - spack.hooks.post_install(self) + # Once everything else is done, run post install hooks + spack.hooks.post_install(self) def _sanity_check_install(self): From 240ada5775c7857932279d86e4305ef001d33717 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:51:09 -0800 Subject: [PATCH 201/339] Add `expand=False` option for URL downloads. - Allows skipping the expand step for downloads. - Fixed stage so that it knows expansion didn't fail when there is a no-expand URLFetchStrategy. - Updated docs to reflect new option, and provided an example. --- lib/spack/docs/packaging_guide.rst | 29 ++++++++++++++++++ lib/spack/llnl/util/filesystem.py | 9 +++++- lib/spack/spack/fetch_strategy.py | 7 ++++- lib/spack/spack/mirror.py | 13 ++++++-- lib/spack/spack/stage.py | 48 ++++++++++++++++++++---------- 5 files changed, 86 insertions(+), 20 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..bae8c34d522 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -401,6 +401,35 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +Skipping the expand step +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack normally expands archives automatically after downloading +them. If you want to skip this step (e.g., for self-extracting +executables and other custom archive types), you can add +``expand=False`` to a ``version`` directive. + +.. code-block:: python + + version('8.2.1', '4136d7b4c04df68b686570afa26988ac', + url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False') + +When ``expand`` is set to ``False``, Spack sets the current working +directory to the directory containing the downloaded archive before it +calls your ``install`` method. Within ``install``, the path to the +downloaded archive is available as ``self.stage.archive_file``. + +Here is an example snippet for packages distribuetd as self-extracting +archives. The example sets permissions on the downloaded file to make +it executable, then runs it with some arguments. + +.. code-block:: python + + def install(self, spec, prefix): + set_executable(self.stage.archive_file) + installer = Executable(self.stage.archive_file) + installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') + Checksums ~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index a92cb0706d2..f218b7c424b 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,8 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -345,6 +346,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + +def set_executable(path): + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + + def remove_dead_links(root): """ Removes any dead link that is present in root diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index ec17cb97f14..0d0a7db8a91 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -82,7 +82,6 @@ class FetchStrategy(object): class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" - def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) @@ -145,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs): self.digest = kwargs.get('md5', None) if not self.digest: self.digest = digest + self.expand_archive = kwargs.get('expand', True) + if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") @@ -218,6 +219,10 @@ def archive_file(self): @_needs_stage def expand(self): + if not self.expand_archive: + tty.msg("Skipping expand step for %s" % self.archive_file) + return + tty.msg("Staging archive: %s" % self.archive_file) self.stage.chdir() diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index fdc4e7967f5..6981f69ac0e 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher): raise ValueError("mirror.path requires spec with concrete version.") if isinstance(fetcher, fs.URLFetchStrategy): - # If we fetch this version with a URLFetchStrategy, use URL's archive type - ext = url.downloaded_file_extension(fetcher.url) + if fetcher.expand_archive: + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = url.downloaded_file_extension(fetcher.url) + else: + # If the archive shouldn't be expanded, don't check for its extension. + ext = None else: # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - return "%s-%s.%s" % (spec.package.name, spec.version, ext) + filename = "%s-%s" % (spec.package.name, spec.version) + if ext: + filename += ".%s" % ext + return filename def mirror_archive_path(spec, fetcher): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..b405915a752 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -229,13 +229,22 @@ def archive_file(self): @property def source_path(self): - """Returns the path to the expanded/checked out source code - within this fetch strategy's path. + """Returns the path to the expanded/checked out source code. - This assumes nothing else is going ot be put in the - FetchStrategy's path. It searches for the first - subdirectory of the path it can find, then returns that. + To find the source code, this method searches for the first + subdirectory of the stage that it can find, and returns it. + This assumes nothing besides the archive file will be in the + stage path, but it has the advantage that we don't need to + know the name of the archive or its contents. + + If the fetch strategy is not supposed to expand the downloaded + file, it will just return the stage path. If the archive needs + to be expanded, it will return None when no archive is found. """ + if isinstance(self.fetcher, fs.URLFetchStrategy): + if not self.fetcher.expand_archive: + return self.path + for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]: if os.path.isdir(p): return p @@ -416,21 +425,15 @@ def expand_archive(self): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and operations that return a value are forwarded to it. """ - - @property - def source_path(self): - return self[0].source_path - - @property - def path(self): - return self[0].path - + # + # __enter__ and __exit__ delegate to all stages in the composite. + # def __enter__(self): for item in self: item.__enter__() @@ -440,9 +443,24 @@ def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): item.__exit__(exc_type, exc_val, exc_tb) + # + # Below functions act only on the *first* stage in the composite. + # + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + def chdir_to_source(self): return self[0].chdir_to_source() + @property + def archive_file(self): + return self[0].archive_file + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" From 1ee90a6fe792bce1730bfd3fddb4fa12b4597361 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 16:52:34 -0800 Subject: [PATCH 202/339] Make spack_cc, spack_cxx, spack_f77, spack_fc available in build env. - Add for convenience for packages that need to refer to these. - Added an example in the documentation. --- lib/spack/docs/packaging_guide.rst | 9 +++++++++ lib/spack/spack/build_environment.py | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bae8c34d522..983adb28b01 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2137,6 +2137,15 @@ Filtering functions Examples: + #. Filtering a Makefile to force it to use Spack's compiler wrappers: + + .. code-block:: python + + filter_file(r'^CC\s*=.*', spack_cc, 'Makefile') + filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile') + filter_file(r'^F77\s*=.*', spack_f77, 'Makefile') + filter_file(r'^FC\s*=.*', spack_fc, 'Makefile') + #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``: .. code-block:: python diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 1b877780801..e22597a789b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -214,6 +214,13 @@ def set_module_variables_for_package(pkg, m): m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + # Put spack compiler paths in module scope. + link_dir = spack.build_env_path + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) + m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) + m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc']) + # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir From 108277fb5dc648c0e93fbd22e6ea6613329acc9d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 6 Mar 2016 19:32:00 -0800 Subject: [PATCH 203/339] Fix docs that have gone stale since repos were added. --- lib/spack/docs/developer_guide.rst | 15 ++++++++++++++- lib/spack/docs/features.rst | 2 +- lib/spack/docs/packaging_guide.rst | 20 ++++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index db47de80f59..0b618aa6830 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -73,19 +73,32 @@ with a high level view of Spack's directory structure:: spack/ <- installation root bin/ spack <- main spack executable + + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. + var/ spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + opt/ spack/ <- packages are installed here + lib/ spack/ docs/ <- source for this documentation env/ <- compiler wrappers for build environment + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries + spack/ <- spack module; contains Python code cmd/ <- each file in here is a spack subcommand compilers/ <- compiler description files - packages/ <- each file in here is a spack package test/ <- unit test modules util/ <- common code diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index fcb810086da..0998ba8da4d 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -103,7 +103,7 @@ creates a simple python file: It doesn't take much python coding to get from there to a working package: -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :lines: 25- Spack also provides wrapper functions around common commands like diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa355..a3e1f4de592 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run If it fails entirely, you can get minimal boilerplate by using :ref:`spack-edit-f`, or you can manually create a directory and - ``package.py`` file for the package in ``var/spack/packages``. + ``package.py`` file for the package in ``var/spack/repos/builtin/packages``. .. note:: @@ -203,7 +203,7 @@ edit`` command: So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides a much simpler shortcut and saves you the trouble of typing the full path. @@ -269,18 +269,18 @@ live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip most of the details here. -``var/spack/packages`` +``var/spack/repos/builtin/packages`` ~~~~~~~~~~~~~~~~~~~~~~~ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, etc.). Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``. -Packages themselves live in ``$SPACK_ROOT/var/spack/packages``. +Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF :shell: :ellipsis: 10 @@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` package lives in:: - $SPACK_ROOT/var/spack/packages/libelf/package.py + $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. @@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. for example, here is -``$SPACK_ROOT/var/spack/packages/libelf/package.py``: +``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python :linenos: @@ -328,7 +328,7 @@ these: $ spack install libelf@0.8.13 Spack sees the package name in the spec and looks for -``libelf/package.py`` in ``var/spack/packages``. Likewise, if you say +``libelf/package.py`` in ``var/spack/repos/builtin/packages``. Likewise, if you say ``spack install py-numpy``, then Spack looks for ``py-numpy/package.py``. @@ -703,7 +703,7 @@ supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory structure like this:: - $SPACK_ROOT/var/spack/packages/ + $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ package.py ad_lustre_rwcontig_open_source.patch @@ -1533,7 +1533,7 @@ The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :start-after: 0.8.12 :linenos: From 265ef337a8cdb7397aa01858077787ca4c2669fb Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 3 Mar 2016 10:40:06 -0500 Subject: [PATCH 204/339] Don't quote -rpath literal --- lib/spack/env/cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c3d11357224..fb0df79d334 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -176,17 +176,17 @@ while [ -n "$1" ]; do -Wl,*) arg="${1#-Wl,}" # TODO: Handle multiple -Wl, continuations of -Wl,-rpath - if [[ $arg == '-rpath='* ]]; then + if [[ $arg == -rpath=* ]]; then arg="${arg#-rpath=}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath,'* ]]; then + elif [[ $arg == -rpath,* ]]; then arg="${arg#-rpath,}" for rpath in ${arg//,/ }; do rpaths+=("$rpath") done - elif [[ $arg == '-rpath' ]]; then + elif [[ $arg == -rpath ]]; then shift; arg="$1" if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" From a399451e1e32c9a294f5b735237cef93ddbd4131 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 6 Mar 2016 23:34:48 -0500 Subject: [PATCH 205/339] Update OpenSSL to 1.0.2g --- var/spack/repos/builtin/packages/openssl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index c73102f05d3..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -17,6 +17,7 @@ class Openssl(Package): version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') + version('1.0.2g', 'f3c710c045cdee5fd114feb69feba7aa') depends_on("zlib") parallel = False From 547933e7e4997a175bf03363b5c0cf7e687f2000 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 01:08:08 -0800 Subject: [PATCH 206/339] Fix bug in -Xlinker argument handling - also update test to match. --- lib/spack/env/cc | 15 ++++++++------- lib/spack/spack/test/cc.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 5e6ed939852..a19346ce971 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -199,19 +199,20 @@ while [ -n "$1" ]; do other_args+=("-Wl,$arg") fi ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi + -Xlinker) + shift; arg="$1"; if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ $arg != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" + if [[ $arg != -Xlinker ]]; then + die "-Xlinker -rpath was not followed by -Xlinker " fi - rpaths+=("${arg#-Xlinker,}") + shift; arg="$1" + rpaths+=("$arg") else - other_args+=("-Xlinker,$arg") + other_args+=("-Xlinker") + other_args+=("$arg") fi ;; *) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 11420ec44a1..f3f6d4a22e6 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -43,7 +43,7 @@ '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath', '-Xlinker,/fourth/rpath', + '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] From 12ddf241a53f85d871c981307e5f682d2e979c75 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Mar 2016 02:39:14 -0800 Subject: [PATCH 207/339] Fix code block in docs. --- lib/spack/spack/multimethod.py | 36 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 51c6a8e89d9..3cd17e796a3 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -138,7 +138,7 @@ class when(object): methods like install() that depend on the package's spec. For example: - .. code-block:: + .. code-block:: python class SomePackage(Package): ... @@ -163,26 +163,28 @@ def install(self, prefix): if you only have part of the install that is platform specific, you could do this: - class SomePackage(Package): - ... - # virtual dependence on MPI. - # could resolve to mpich, mpich2, OpenMPI - depends_on('mpi') + .. code-block:: python - def setup(self): - # do nothing in the default case - pass + class SomePackage(Package): + ... + # virtual dependence on MPI. + # could resolve to mpich, mpich2, OpenMPI + depends_on('mpi') - @when('^openmpi') - def setup(self): - # do something special when this is built with OpenMPI for - # its MPI implementations. + def setup(self): + # do nothing in the default case + pass + + @when('^openmpi') + def setup(self): + # do something special when this is built with OpenMPI for + # its MPI implementations. - def install(self, prefix): - # Do common install stuff - self.setup() - # Do more common install stuff + def install(self, prefix): + # Do common install stuff + self.setup() + # Do more common install stuff There must be one (and only one) @when clause that matches the package's spec. If there is more than one, or if none match, From 1e7d946d9116dee4900b95def1614bd3f788f6f0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 7 Mar 2016 12:37:51 -0600 Subject: [PATCH 208/339] Add patch to allow PGI to build M4 --- var/spack/repos/builtin/packages/m4/inline-pgi.patch | 10 ++++++++++ var/spack/repos/builtin/packages/m4/package.py | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 var/spack/repos/builtin/packages/m4/inline-pgi.patch diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch new file mode 100644 index 00000000000..da5eb57a939 --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/inline-pgi.patch @@ -0,0 +1,10 @@ +--- a/m4/extern-inline.m4 ++++ b/m4/extern-inline.m4 +@@ -34,6 +34,7 @@ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ ++ && !defined __PGI \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) + # define _GL_INLINE inline diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index d6829dbcd4b..ef70add18aa 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,6 +7,8 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + patch('inline-pgi.patch', when='@1.4.17') + variant('sigsegv', default=True, description="Build the libsigsegv dependency") depends_on('libsigsegv', when='+sigsegv') From b7750cf61c51704d87d679efeba9ca8cb8c2b768 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 7 Mar 2016 15:52:22 -0600 Subject: [PATCH 209/339] Autoreconf is necessary after patch --- var/spack/repos/builtin/packages/m4/inline-pgi.patch | 12 ++++++++++++ var/spack/repos/builtin/packages/m4/package.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch index da5eb57a939..bc31ad918ed 100644 --- a/var/spack/repos/builtin/packages/m4/inline-pgi.patch +++ b/var/spack/repos/builtin/packages/m4/inline-pgi.patch @@ -8,3 +8,15 @@ && !(defined __SUNPRO_C && __STDC__))) \ && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) # define _GL_INLINE inline +--- a/configure.ac ++++ b/configure.ac +@@ -22,7 +22,7 @@ + [bug-m4@gnu.org]) + AC_CONFIG_AUX_DIR([build-aux]) + +-AM_INIT_AUTOMAKE([1.11.6 dist-bzip2 dist-xz color-tests parallel-tests ++AM_INIT_AUTOMAKE([dist-bzip2 dist-xz color-tests parallel-tests + silent-rules subdir-objects gnu]) + + m4_pattern_forbid([^M4_[A-Z]]) + diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index ef70add18aa..aa0c775f083 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -14,6 +14,10 @@ class M4(Package): depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): + # After patch, update generated configuration files that depend on extern-inline.m4 + autoreconf = which('autoreconf') + autoreconf() + configure_args = [] if 'libsigsegv' in spec: configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) From 77ec27c73013eb14821a4249f6a08a8321e10eef Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 8 Mar 2016 11:09:41 +0100 Subject: [PATCH 210/339] fixed bug : similar issues in checksum and md5 as were solved in ad103dcafa652a839590f5fce28b2e2ce3b5a56d --- lib/spack/spack/cmd/checksum.py | 39 +++++++++--------------- lib/spack/spack/cmd/md5.py | 54 ++++++++++++++++----------------- lib/spack/spack/cmd/stage.py | 4 ++- lib/spack/spack/stage.py | 1 + 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 966ff9a5e96..5504673c9e4 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -22,23 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re import argparse import hashlib -from pprint import pprint -from subprocess import CalledProcessError import llnl.util.tty as tty -from llnl.util.tty.colify import colify - import spack import spack.cmd import spack.util.crypto from spack.stage import Stage, FailedDownloadError from spack.version import * -description ="Checksum available versions of a package." +description = "Checksum available versions of a package." + def setup_parser(subparser): subparser.add_argument( @@ -60,30 +55,24 @@ def get_checksums(versions, urls, **kwargs): hashes = [] i = 0 for url, version in zip(urls, versions): - stage = Stage(url) try: - stage.fetch() - if i == 0 and first_stage_function: - first_stage_function(stage) + with Stage(url) as stage: + stage.delete_on_exit = not keep_stage + stage.fetch() + if i == 0 and first_stage_function: + first_stage_function(stage) - hashes.append((version, - spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) - except FailedDownloadError, e: + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + i += 1 + except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) - continue - except Exception, e: + except Exception as e: tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) - continue - - finally: - if not keep_stage: - stage.destroy() - i += 1 return hashes - def checksum(parser, args): # get the package we're going to generate checksums for pkg = spack.repo.get(args.package) @@ -106,8 +95,8 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( - ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print + ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) + print() archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 879ef9f7b74..20508abf99c 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -22,51 +22,51 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import argparse import hashlib - -from contextlib import contextmanager +import os import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack.util.crypto from spack.stage import Stage, FailedDownloadError description = "Calculate md5 checksums for files/urls." -@contextmanager -def stager(url): - _cwd = os.getcwd() - _stager = Stage(url) - try: - _stager.fetch() - yield _stager - except FailedDownloadError: - tty.msg("Failed to fetch %s" % url) - finally: - _stager.destroy() - os.chdir(_cwd) # the Stage class changes the current working dir so it has to be restored def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, help="Files to checksum.") + +def compute_md5_checksum(url): + if not os.path.isfile(url): + with Stage(url) as stage: + stage.fetch() + value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + else: + value = spack.util.crypto.checksum(hashlib.md5, url) + return value + + def md5(parser, args): if not args.files: setup_parser.parser.print_help() return 1 - for f in args.files: - if not os.path.isfile(f): - with stager(f) as stage: - checksum = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) - print "%s %s" % (checksum, f) - else: - if not can_access(f): - tty.die("Cannot read file: %s" % f) + results = [] + for url in args.files: + try: + checksum = compute_md5_checksum(url) + results.append((checksum, url)) + except FailedDownloadError as e: + tty.warn("Failed to fetch %s" % url) + tty.warn("%s" % e) + except IOError as e: + tty.warn("Error when reading %s" % url) + tty.warn("%s" % e) - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + # Dump the MD5s at last without interleaving them with downloads + tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + for checksum, url in results: + tty.msg("%s %s" % (checksum, url)) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 5786780efb4..b575f6c4566 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,4 +50,6 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - package.do_stage() + with package.stage as stage: + stage.delete_on_exit = False + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b117c76aa17..9404e12e849 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -438,6 +438,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): + item.delete_on_exit = getattr(self, 'delete_on_exit', True) item.__exit__(exc_type, exc_val, exc_tb) def chdir_to_source(self): From 18ce5ccf8fb31dae71848f541f3e139bd582953c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:49:11 -0800 Subject: [PATCH 211/339] Avoid race in pango's `make install`, set parallel=False --- var/spack/repos/builtin/packages/pango/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py index df43625bf5c..79dad3a3d2f 100644 --- a/var/spack/repos/builtin/packages/pango/package.py +++ b/var/spack/repos/builtin/packages/pango/package.py @@ -16,4 +16,4 @@ class Pango(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() - make("install") + make("install", parallel=False) From 5aadb6df19ea6d7575ee5f5e5f18d92934245d98 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 02:50:26 -0800 Subject: [PATCH 212/339] Fixes #502. Create install prefix before build, clean up do_install. - Fix bug introduced during merge of stage refactor. - install prefix was not created before build_environment.fork() - build_environment.fork() calls setup_dependent_environment - python's setup_dependent_environment can inadvertently create the install prefix before directory_layout expects it. - Clean up Package.do_install: - simplify control flow: parent process now entirely responsible for creating/destroying the install prefix. cleanup is now in one place. - Hoisting cleanup out of the child improves nesting of try/catch in `real_work`. - `real_work` renamed to `build_process` --- lib/spack/spack/directory_layout.py | 56 +++++++++----- lib/spack/spack/package.py | 115 +++++++++++++--------------- 2 files changed, 91 insertions(+), 80 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 08c23627f46..242eb1afa04 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -85,6 +85,16 @@ def create_install_directory(self, spec): raise NotImplementedError() + def check_installed(self, spec): + """Checks whether a spec is installed. + + Return the spec's prefix, if it is installed, None otherwise. + + Raise an exception if the install is inconsistent or corrupt. + """ + raise NotImplementedError() + + def extension_map(self, spec): """Get a dict of currently installed extension packages for a spec. @@ -246,26 +256,36 @@ def build_packages_path(self, spec): def create_install_directory(self, spec): _check_concrete(spec) + prefix = self.check_installed(spec) + if prefix: + raise InstallDirectoryAlreadyExistsError(prefix) + + mkdirp(self.metadata_path(spec)) + self.write_spec(spec, self.spec_file_path(spec)) + + + def check_installed(self, spec): + _check_concrete(spec) path = self.path_for_spec(spec) spec_file_path = self.spec_file_path(spec) - if os.path.isdir(path): - if not os.path.isfile(spec_file_path): - raise InconsistentInstallDirectoryError( - 'No spec file found at path %s' % spec_file_path) + if not os.path.isdir(path): + return None - installed_spec = self.read_spec(spec_file_path) - if installed_spec == self.spec: - raise InstallDirectoryAlreadyExistsError(path) + if not os.path.isfile(spec_file_path): + raise InconsistentInstallDirectoryError( + 'Inconsistent state: install prefix exists but contains no spec.yaml:', + " " + path) - if spec.dag_hash() == installed_spec.dag_hash(): - raise SpecHashCollisionError(installed_hash, spec_hash) - else: - raise InconsistentInstallDirectoryError( - 'Spec file in %s does not match hash!' % spec_file_path) + installed_spec = self.read_spec(spec_file_path) + if installed_spec == spec: + return path - mkdirp(self.metadata_path(spec)) - self.write_spec(spec, spec_file_path) + if spec.dag_hash() == installed_spec.dag_hash(): + raise SpecHashCollisionError(installed_hash, spec_hash) + else: + raise InconsistentInstallDirectoryError( + 'Spec file in %s does not match hash!' % spec_file_path) def all_specs(self): @@ -399,8 +419,8 @@ def remove_extension(self, spec, ext_spec): class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" - def __init__(self, message): - super(DirectoryLayoutError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(DirectoryLayoutError, self).__init__(message, long_msg) class SpecHashCollisionError(DirectoryLayoutError): @@ -422,8 +442,8 @@ def __init__(self, installed_spec, prefix, error): class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" - def __init__(self, message): - super(InconsistentInstallDirectoryError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(InconsistentInstallDirectoryError, self).__init__(message, long_msg) class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d4acbf50243..972a0410b9a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -845,7 +845,8 @@ def do_install(self, if not self.spec.concrete: raise ValueError("Can only install concrete packages.") - if os.path.exists(self.prefix): + # Ensure package is not already installed + if spack.install_layout.check_installed(self.spec): tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return @@ -857,18 +858,11 @@ def do_install(self, keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) - def cleanup(): - """Handles removing install prefix on error.""" - if not keep_prefix: - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed. " + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # Set parallelism before starting build. + self.make_jobs = make_jobs # Then install the package itself. - def real_work(): + def build_process(): """Forked for each build. Has its own process and python module space set up by build_environment.fork().""" start_time = time.time() @@ -878,30 +872,24 @@ def real_work(): else: self.do_stage() - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) + tty.msg("Building %s" % self.name) - try: - tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) - self.stage.keep = keep_stage - with self.stage: - # Run the pre-install hook in the child process after - # the directory is created. - spack.hooks.pre_install(self) + if fake: + self.do_fake_install() + else: + # Do the real install in the source directory. + self.stage.chdir_to_source() - if fake: - self.do_fake_install() - - else: - # Do the real install in the source directory. - self.stage.chdir_to_source() - - # Save the build environment in a file before building. - env_path = join_path(os.getcwd(), 'spack-build.env') + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') + try: # Redirect I/O to a build log (and optionally to the terminal) log_path = join_path(os.getcwd(), 'spack-build.out') log_file = open(log_path, 'w') @@ -909,43 +897,46 @@ def real_work(): dump_environment(env_path) self.install(self.spec, self.prefix) - # Ensure that something was actually installed. - self._sanity_check_install() + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Copy provenance into the install directory on success - log_install_path = spack.install_layout.build_log_path(self.spec) - env_install_path = spack.install_layout.build_env_path(self.spec) - packages_dir = spack.install_layout.build_packages_path(self.spec) + # Ensure that something was actually installed. + self._sanity_check_install() - install(log_path, log_install_path) - install(env_path, env_install_path) - dump_packages(self.spec, packages_dir) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) - tty.msg("Successfully installed %s" % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - except ProcessError as e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - except: - # other exceptions just clean up and raise. - cleanup() - raise - - # Set parallelism before starting build. - self.make_jobs = make_jobs - - # Do the build. - spack.build_environment.fork(self, real_work) + try: + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) + spack.build_environment.fork(self, build_process) + except: + # remove the install prefix if anything went wrong during install. + if not keep_prefix: + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. From e0e545774aaddb5386367bd900a31b85692abc51 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 8 Mar 2016 12:15:40 -0600 Subject: [PATCH 213/339] Add more versions to Autotools --- var/spack/repos/builtin/packages/autoconf/package.py | 1 + var/spack/repos/builtin/packages/automake/package.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py index 5189faf0548..6412e810a6a 100644 --- a/var/spack/repos/builtin/packages/autoconf/package.py +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -6,6 +6,7 @@ class Autoconf(Package): url = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz" version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b') + version('2.62', '6c1f3b3734999035d77da5024aab4fbd') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py index 9115822730e..2172a42030f 100644 --- a/var/spack/repos/builtin/packages/automake/package.py +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -5,7 +5,9 @@ class Automake(Package): homepage = "http://www.gnu.org/software/automake/" url = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz" + version('1.15', '716946a105ca228ab545fc37a70df3a3') version('1.14.1', 'd052a3e884631b9c7892f2efce542d75') + version('1.11.6', '0286dc30295b62985ca51919202ecfcc') depends_on('autoconf') From a4861a30729e32aef493b3418d03874837eecc08 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 10:39:46 -0800 Subject: [PATCH 214/339] Minor changes/bugfixes on md5/checksum PR --- lib/spack/spack/cmd/checksum.py | 5 ++--- lib/spack/spack/cmd/md5.py | 4 ++-- lib/spack/spack/cmd/stage.py | 4 +--- lib/spack/spack/stage.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 5504673c9e4..518d2703dcd 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -56,8 +56,7 @@ def get_checksums(versions, urls, **kwargs): i = 0 for url, version in zip(urls, versions): try: - with Stage(url) as stage: - stage.delete_on_exit = not keep_stage + with Stage(url, keep=keep_stage) as stage: stage.fetch() if i == 0 and first_stage_function: first_stage_function(stage) @@ -96,7 +95,7 @@ def checksum(parser, args): tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) - print() + print archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 20508abf99c..f99fc0f8c2b 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -67,6 +67,6 @@ def md5(parser, args): tty.warn("%s" % e) # Dump the MD5s at last without interleaving them with downloads - tty.msg("Number of MD5 check-sums computed: %s " % len(results)) + tty.msg("%d MD5 checksums:" % len(results)) for checksum, url in results: - tty.msg("%s %s" % (checksum, url)) + print "%s %s" % (checksum, url) diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index b575f6c4566..5786780efb4 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -50,6 +50,4 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: package = spack.repo.get(spec) - with package.stage as stage: - stage.delete_on_exit = False - package.do_stage() + package.do_stage() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 0d35511c347..d2ed03c271e 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -441,7 +441,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.delete_on_exit = getattr(self, 'delete_on_exit', True) + item.keep = getattr(self, 'keep', None) item.__exit__(exc_type, exc_val, exc_tb) # From 0c7d0c0b6c650cd0fd6f4814a255613e3efa1814 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Tue, 8 Mar 2016 12:26:40 -0800 Subject: [PATCH 215/339] Variants and fixes to thrift package --- .../repos/builtin/packages/thrift/package.py | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index 0e15052f644..ec3b13e5632 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -12,33 +12,45 @@ class Thrift(Package): version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') - extends("python") + # Currently only support for c-family and python + variant('c', default=True, description="Build support for C-family languages") + variant('python', default=True, description="Build support for python") - depends_on("autoconf") - depends_on("automake") - depends_on("bison") - depends_on("boost") - depends_on("flex") - depends_on("jdk") - depends_on("libtool") - depends_on("openssl") - depends_on("python") + depends_on('jdk') + depends_on('autoconf') + depends_on('automake') + depends_on('libtool') + depends_on('boost@1.53:') + depends_on('bison') + depends_on('flex') + depends_on('openssl') + + # Variant dependencies + extends('python', when='+python') + depends_on('python', when='+python') + + depends_on('zlib', when='+c') + depends_on('libevent', when='+c') - # Compilation fails for most languages, fortunately cpp installs fine - # All other languages (yes, including C) are omitted until someone needs them def install(self, spec, prefix): - env["PY_PREFIX"] = prefix - env["JAVA_PREFIX"] = prefix + env['PY_PREFIX'] = prefix + env['JAVA_HOME'] = spec['jdk'].prefix - configure("--prefix=%s" % prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-c=no", - "--with-go=no", - "--with-python=yes", - "--with-lua=no", - "--with-php=no", - "--with-qt4=no", - "--enable-tests=no") + # configure options + options = ['--prefix=%s' % prefix] + + options.append('--with-boost=%s' % spec['boost'].prefix) + options.append('--enable-tests=no') + + options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no')) + options.append('--with-python=%s' % ('yes' if '+python' in spec else 'no')) + options.append('--with-java=%s' % ('yes' if '+java' in spec else 'no')) + options.append('--with-go=%s' % ('yes' if '+go' in spec else 'no')) + options.append('--with-lua=%s' % ('yes' if '+lua' in spec else 'no')) + options.append('--with-php=%s' % ('yes' if '+php' in spec else 'no')) + options.append('--with-qt4=%s' % ('yes' if '+qt4' in spec else 'no')) + + configure(*options) make() make("install") From a1be45d0e7f1921176b40b2aa497309029f1f7ad Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:56:44 -0800 Subject: [PATCH 216/339] Fix bug with setting module-scope vars in derived package classes. --- lib/spack/spack/build_environment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e22597a789b..87fc310b5a3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -177,8 +177,6 @@ def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - m = pkg.module - # number of jobs spack will to build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: From df84677d1640139673c26a4c407698486905fbb3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:57:13 -0800 Subject: [PATCH 217/339] Make diy generate verbose build output by default. - added -q option to shut it up --- lib/spack/spack/cmd/diy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9df53312f80..2c3a8761ab6 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -45,6 +45,9 @@ def setup_parser(subparser): subparser.add_argument( '--skip-patch', action='store_true', help="Skip patching for the DIY build.") + subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -92,4 +95,5 @@ def diy(self, args): package.do_install( keep_prefix=args.keep_prefix, ignore_deps=args.ignore_deps, + verbose=not args.quiet, keep_stage=True) # don't remove source dir for DIY. From 9c6184373752746e1219263c1afd9d1955911891 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 8 Mar 2016 13:58:41 -0800 Subject: [PATCH 218/339] Fix bugs in DIYStage: fetch & context handling. - DIYStage needs to be a context handler - DIYStage.fetch needs to take 2 args. --- lib/spack/spack/stage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index d2ed03c271e..5354135e6a3 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -477,10 +477,14 @@ def chdir(self): else: raise ChdirError("Setup failed: no such directory: " + self.path) + # DIY stages do nothing as context managers. + def __enter__(self): pass + def __exit__(self, exc_type, exc_val, exc_tb): pass + def chdir_to_source(self): self.chdir() - def fetch(self): + def fetch(self, mirror_only): tty.msg("No need to fetch for DIY.") def check(self): From 5b22873b3dce37c0d7bad418bf6e0b73d381d19e Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 00:18:20 -0500 Subject: [PATCH 219/339] Fixed issues with hypre: 1. --with-lapack-lib was wrong. 2. --with-MPI was wrong; set env vars for MPI wrappers instead. 3. Added version 2.10.1 4. Added shared library variant (True by default). Hypre can build shared or static libraries, but not both in the same build. --- .../repos/builtin/packages/hypre/package.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 0f7f14dd89f..242ee100d7a 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Hypre(Package): """Hypre is a library of high performance preconditioners that @@ -8,8 +9,11 @@ class Hypre(Package): homepage = "http://computation.llnl.gov/project/linear_solvers/software.php" url = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + version('2.10.1', 'dc048c4cabb3cd549af72591474ad674') version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') + variant('shared', default=True, description="Build shared library version (disables static library)") + depends_on("mpi") depends_on("blas") depends_on("lapack") @@ -17,16 +21,26 @@ class Hypre(Package): def install(self, spec, prefix): blas_dir = spec['blas'].prefix lapack_dir = spec['lapack'].prefix + mpi_dir = spec['mpi'].prefix + + os.environ['CC'] = os.path.join(mpi_dir, 'bin', 'mpicc') + os.environ['CXX'] = os.path.join(mpi_dir, 'bin', 'mpicxx') + os.environ['F77'] = os.path.join(mpi_dir, 'bin', 'mpif77') + + + configure_args = [ + "--prefix=%s" % prefix, + "--with-lapack-libs=lapack", + "--with-lapack-lib-dirs=%s/lib" % lapack_dir, + "--with-blas-libs=blas", + "--with-blas-lib-dirs=%s/lib" % blas_dir] + if '+shared' in self.spec: + configure_args.append("--enable-shared") # Hypre's source is staged under ./src so we'll have to manually # cd into it. with working_dir("src"): - configure( - "--prefix=%s" % prefix, - "--with-blas-libs=blas", - "--with-blas-lib-dirs=%s/lib" % blas_dir, - "--with-lapack-libs=\"lapack blas\"", - "--with-lapack-lib-dirs=%s/lib" % lapack_dir, - "--with-MPI") + configure(*configure_args) + make() make("install") From c67b922185ebf5b7686d5b4958e2b31b8f0f1469 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 00:34:08 -0500 Subject: [PATCH 220/339] Added shared library capability. --- var/spack/repos/builtin/packages/petsc/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 87f700629df..e42425bcefa 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -11,8 +11,11 @@ class Petsc(Package): version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') - depends_on("python @2.6:2.9") # requires Python for building + variant('shared', default=True, description="Build shared library version") + + depends_on("python @2.6:2.7") # requires Python for building depends_on("boost") depends_on("blas") @@ -33,7 +36,7 @@ def install(self, spec, prefix): "--with-metis-dir=%s" % spec['metis'].prefix, "--with-hdf5-dir=%s" % spec['hdf5'].prefix, "--with-mpi-dir=%s" % spec['mpi'].prefix, - "--with-shared-libraries=0") + "--with-shared-libraries=%d" % (1 if '+shared' in self.spec else 0)) # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) From f01d1c4385f05dd7bda7efb3a9e7f2c8008e177a Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 9 Mar 2016 14:36:37 +0100 Subject: [PATCH 221/339] petsc : added variants and logic to build various flavors --- .../repos/builtin/packages/petsc/package.py | 92 ++++++++++++++----- 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 87f700629df..5be187f3487 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -1,40 +1,86 @@ +import os from spack import * + class Petsc(Package): - """PETSc is a suite of data structures and routines for the - scalable (parallel) solution of scientific applications modeled by - partial differential equations.""" + """ + PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications + modeled by partial differential equations. + """ homepage = "http://www.mcs.anl.gov/petsc/index.html" - url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + version('3.6.3', '91dd3522de5a5ef039ff8f50800db606') version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') - depends_on("python @2.6:2.9") # requires Python for building + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + variant('double', default=True, description='Switches between single and double precision') - depends_on("boost") - depends_on("blas") - depends_on("lapack") - depends_on("hypre") - depends_on("parmetis") - depends_on("metis") - depends_on("hdf5+mpi") - depends_on("mpi") + variant('metis', default=True, description='Activates support for metis and parmetis') + variant('hdf5', default=True, description='Activates support for HDF5 (only parallel)') + variant('boost', default=True, description='Activates support for Boost') + variant('hypre', default=True, description='Activates support for Hypre') + + # Build dependencies + depends_on('python @2.6:2.9') # requires Python for building + + # Virtual dependencies + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + # Other dependencies + depends_on('boost', when='+boost') + depends_on('metis', when='+metis') + + depends_on('hdf5~cxx~unsupported+mpi', when='+hdf5+mpi') + depends_on('parmetis', when='+metis+mpi') + depends_on('hypre', when='+hypre+mpi') + + def mpi_dependent_options(self): + if '~mpi' in self.spec: + compiler_opts = [ + '--with-cc=%s' % os.environ['CC'], + '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), + '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), + '--with-mpi=0' + ] + error_message_fmt = '\t{library} support requires "+mpi" to be activated' + errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre') if ('+'+x) in self.spec] + if errors: + errors = ['incompatible variants given'] + errors + raise RuntimeError('\n'.join(errors)) + else: + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] + return compiler_opts def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, - "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, - "--with-boost-dir=%s" % spec['boost'].prefix, - "--with-hypre-dir=%s" % spec['hypre'].prefix, - "--with-parmetis-dir=%s" % spec['parmetis'].prefix, - "--with-metis-dir=%s" % spec['metis'].prefix, - "--with-hdf5-dir=%s" % spec['hdf5'].prefix, - "--with-mpi-dir=%s" % spec['mpi'].prefix, - "--with-shared-libraries=0") + options = [] + options.extend(self.mpi_dependent_options()) + options.extend([ + '--with-precision=%s' % ('double' if '+double' in spec else 'single'), + '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), + '--with-blas-lapack-dir=%s' % spec['lapack'].prefix + ]) + # Activates library support if needed + for library in ('metis', 'boost', 'hfd5', 'hypre', 'parmetis'): + options.append( + '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) + ) + if library in spec: + options.append( + '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) + ) + + configure('--prefix=%s' % prefix, *options) # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) make("install") From 8e76cda200bb3da159cdc27726c7812f66c5a5ed Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 9 Mar 2016 10:41:31 -0600 Subject: [PATCH 222/339] Patch config.hin, not extern-inline.m4 --- .../builtin/packages/m4/inline-pgi.patch | 22 ------------------- .../repos/builtin/packages/m4/package.py | 6 +---- var/spack/repos/builtin/packages/m4/pgi.patch | 10 +++++++++ 3 files changed, 11 insertions(+), 27 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/m4/inline-pgi.patch create mode 100644 var/spack/repos/builtin/packages/m4/pgi.patch diff --git a/var/spack/repos/builtin/packages/m4/inline-pgi.patch b/var/spack/repos/builtin/packages/m4/inline-pgi.patch deleted file mode 100644 index bc31ad918ed..00000000000 --- a/var/spack/repos/builtin/packages/m4/inline-pgi.patch +++ /dev/null @@ -1,22 +0,0 @@ ---- a/m4/extern-inline.m4 -+++ b/m4/extern-inline.m4 -@@ -34,6 +34,7 @@ - ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ - : (199901L <= __STDC_VERSION__ \ - && !defined __HP_cc \ -+ && !defined __PGI \ - && !(defined __SUNPRO_C && __STDC__))) \ - && !defined _GL_EXTERN_INLINE_STDHEADER_BUG) - # define _GL_INLINE inline ---- a/configure.ac -+++ b/configure.ac -@@ -22,7 +22,7 @@ - [bug-m4@gnu.org]) - AC_CONFIG_AUX_DIR([build-aux]) - --AM_INIT_AUTOMAKE([1.11.6 dist-bzip2 dist-xz color-tests parallel-tests -+AM_INIT_AUTOMAKE([dist-bzip2 dist-xz color-tests parallel-tests - silent-rules subdir-objects gnu]) - - m4_pattern_forbid([^M4_[A-Z]]) - diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py index aa0c775f083..a4b9dcb6231 100644 --- a/var/spack/repos/builtin/packages/m4/package.py +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -7,17 +7,13 @@ class M4(Package): version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') - patch('inline-pgi.patch', when='@1.4.17') + patch('pgi.patch', when='@1.4.17') variant('sigsegv', default=True, description="Build the libsigsegv dependency") depends_on('libsigsegv', when='+sigsegv') def install(self, spec, prefix): - # After patch, update generated configuration files that depend on extern-inline.m4 - autoreconf = which('autoreconf') - autoreconf() - configure_args = [] if 'libsigsegv' in spec: configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) diff --git a/var/spack/repos/builtin/packages/m4/pgi.patch b/var/spack/repos/builtin/packages/m4/pgi.patch new file mode 100644 index 00000000000..1ad63e2cf1c --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/pgi.patch @@ -0,0 +1,10 @@ +--- a/lib/config.hin ++++ b/lib/config.hin +@@ -1510,6 +1510,7 @@ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ ++ && !defined __PGI \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_APPLE_BUG) + # define _GL_INLINE inline From bf4d51ea0b00e091728ecf8d4d51aa295448a416 Mon Sep 17 00:00:00 2001 From: Luigi Calori Date: Wed, 9 Mar 2016 14:45:29 +0100 Subject: [PATCH 223/339] fix error exit when doing spack patch without parameters, same as spack stage --- lib/spack/spack/cmd/patch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 44fc8696db3..b04b4027384 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -24,6 +24,7 @@ ############################################################################## import argparse +import llnl.util.tty as tty import spack.cmd import spack From b43c277dc617020d7aaa1e292e01e79d0840999e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 10:55:50 -0800 Subject: [PATCH 224/339] Merge @citibeth and @alalazo's petsc fixes from #515 and #517 --- .../repos/builtin/packages/petsc/package.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 5be187f3487..41fd8599452 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -15,6 +15,7 @@ class Petsc(Package): version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=True, description='Activates MPI support') @@ -25,21 +26,21 @@ class Petsc(Package): variant('boost', default=True, description='Activates support for Boost') variant('hypre', default=True, description='Activates support for Hypre') - # Build dependencies - depends_on('python @2.6:2.9') # requires Python for building - # Virtual dependencies depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') + # Build dependencies + depends_on('python @2.6:2.7') + # Other dependencies depends_on('boost', when='+boost') depends_on('metis', when='+metis') - depends_on('hdf5~cxx~unsupported+mpi', when='+hdf5+mpi') + depends_on('hdf5+mpi', when='+hdf5+mpi') depends_on('parmetis', when='+metis+mpi') - depends_on('hypre', when='+hypre+mpi') + depends_on('hypre', when='+hypre+mpi') def mpi_dependent_options(self): if '~mpi' in self.spec: @@ -50,7 +51,9 @@ def mpi_dependent_options(self): '--with-mpi=0' ] error_message_fmt = '\t{library} support requires "+mpi" to be activated' - errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre') if ('+'+x) in self.spec] + errors = [error_message_fmt.format(library=x) + for x in ('hdf5', 'hypre', 'parmetis') + if ('+'+x) in self.spec] if errors: errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) @@ -70,7 +73,7 @@ def install(self, spec, prefix): '--with-blas-lapack-dir=%s' % spec['lapack'].prefix ]) # Activates library support if needed - for library in ('metis', 'boost', 'hfd5', 'hypre', 'parmetis'): + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis'): options.append( '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) ) @@ -79,8 +82,8 @@ def install(self, spec, prefix): '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) ) - configure('--prefix=%s' % prefix, *options) + # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) make("install") From d06ebf23d4ffe3499edca3f34f60be9b561f5f8c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 11:16:35 -0800 Subject: [PATCH 225/339] Removing `unsupported` variant from HDF5. - `unsupported` shouldn't be a variant. --- var/spack/repos/builtin/packages/hdf5/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index ed4e7c35c99..513a38ee8a4 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -46,7 +46,6 @@ class Hdf5(Package): variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') - variant('unsupported', default=True, description='Enables unsupported configuration options') variant('mpi', default=False, description='Enable MPI support') variant('szip', default=False, description='Enable szip support') @@ -74,6 +73,13 @@ def install(self, spec, prefix): self.validate(spec) # Handle compilation after spec validation extra_args = [] + + # Always enable this option. This does not actually enable any + # features: it only *allows* the user to specify certain + # combinations of other arguments. Enabling it just skips a + # sanity check in configure, so this doesn't merit a variant. + extra_args.append("--enable-unsupported") + if '+debug' in spec: extra_args.append('--enable-debug=all') else: @@ -84,9 +90,6 @@ def install(self, spec, prefix): else: extra_args.append('--enable-static-exec') - if '+unsupported' in spec: - extra_args.append("--enable-unsupported") - if '+cxx' in spec: extra_args.append('--enable-cxx') From a4d40177fdfece3c25d7ebb48855e8fee2a7093d Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 8 Mar 2016 09:44:21 -0700 Subject: [PATCH 226/339] + Provide download/build instructions for qt/5.4.2. - This version provides updates to provided cmake scripts that are required for building cmake-gui. --- var/spack/repos/builtin/packages/qt/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..ef5f05601f8 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -8,6 +8,9 @@ class Qt(Package): list_url = 'http://download.qt-project.org/official_releases/qt/' list_depth = 2 + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') From 383e73a5f53e0ab3d1f448789bc5a1b4a7c85292 Mon Sep 17 00:00:00 2001 From: Alfredo Adolfo Gimenez Date: Wed, 9 Mar 2016 11:25:51 -0800 Subject: [PATCH 227/339] Remove unneccessary depends_on --- var/spack/repos/builtin/packages/thrift/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index ec3b13e5632..6430f40e80d 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -27,7 +27,6 @@ class Thrift(Package): # Variant dependencies extends('python', when='+python') - depends_on('python', when='+python') depends_on('zlib', when='+c') depends_on('libevent', when='+c') From 45ef496dd553e023903b8b4d5bcace59c56eb486 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 11:26:37 -0800 Subject: [PATCH 228/339] Add some descriptive language to a list comprehension. --- var/spack/repos/builtin/packages/petsc/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 41fd8599452..efe172fc083 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -51,6 +51,9 @@ def mpi_dependent_options(self): '--with-mpi=0' ] error_message_fmt = '\t{library} support requires "+mpi" to be activated' + + # If mpi is disabled (~mpi), it's an error to have any of these enabled. + # This generates a list of any such errors. errors = [error_message_fmt.format(library=x) for x in ('hdf5', 'hypre', 'parmetis') if ('+'+x) in self.spec] From 6e82ab1f1501c8b64383f1d9a26d743284f4054f Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 9 Mar 2016 21:18:44 +0100 Subject: [PATCH 229/339] change of url for mpfr --- var/spack/repos/builtin/packages/mpfr/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py index a1bd7529cfe..7e6e7d5bb6f 100644 --- a/var/spack/repos/builtin/packages/mpfr/package.py +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -28,8 +28,9 @@ class Mpfr(Package): """The MPFR library is a C library for multiple-precision floating-point computations with correct rounding.""" homepage = "http://www.mpfr.org" - url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" + url = "https://gforge.inria.fr/frs/download.php/latestfile/159/mpfr-3.1.2.tar.bz2" + version('3.1.4', 'b8a2f6b0e68bef46e53da2ac439e1cf4') version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') From ca102295657adef7100f638be0dabc0e6d7cf37f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 14:56:21 -0800 Subject: [PATCH 230/339] Fixes #524 - Had attempted to add more functionality by assigning different meanign None, True, and False values "keep_stage" (where False was "always delete"). - Turns out that's not really worth the complexity. Having the third "always delete" sense is hardly ever useful but makes the code hard to understand. --- lib/spack/spack/package.py | 7 ++++--- lib/spack/spack/stage.py | 24 ++++++++++-------------- lib/spack/spack/test/stage.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 972a0410b9a..ca9e9c4bd1d 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -825,7 +825,7 @@ def _resource_stage(self, resource): def do_install(self, - keep_prefix=False, keep_stage=None, ignore_deps=False, + keep_prefix=False, keep_stage=False, ignore_deps=False, skip_patch=False, verbose=False, make_jobs=None, fake=False): """Called by commands to install a package and its dependencies. @@ -834,8 +834,9 @@ def do_install(self, Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Set to True or false to always keep or always delete stage. - By default, stage is destroyed only if there are no exceptions. + keep_stage -- By default, stage is destroyed only if there are no + exceptions during build. Set to True to keep the stage + even with exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 5354135e6a3..f88f82fc2d6 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -88,7 +88,8 @@ class Stage(object): similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None): + def __init__(self, url_or_fetch_strategy, + name=None, mirror_path=None, keep=False): """Create a stage object. Parameters: url_or_fetch_strategy @@ -108,10 +109,9 @@ def __init__(self, url_or_fetch_strategy, name=None, mirror_path=None, keep=None keep By default, when used as a context manager, the Stage - is cleaned up when everything goes well, and it is - kept intact when an exception is raised. You can - override this behavior by setting keep to True - (always keep) or False (always delete). + is deleted on exit when no exceptions are raised. + Pass True to keep the stage intact even if no + exceptions are raised. """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -166,12 +166,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): Returns: Boolean """ - if self.keep is None: - # Default: delete when there are no exceptions. - if exc_type is None: self.destroy() - - elif not self.keep: - # Overridden. Either always keep or always delete. + # Delete when there are no exceptions, unless asked to keep. + if exc_type is None and not self.keep: self.destroy() @@ -195,8 +191,8 @@ def _need_to_create_path(self): real_tmp = os.path.realpath(self.tmp_root) if spack.use_tmp_stage: - # If we're using a tmp dir, it's a link, and it points at the right spot, - # then keep it. + # If we're using a tmp dir, it's a link, and it points at the + # right spot, then keep it. if (real_path.startswith(real_tmp) and os.path.exists(real_path)): return False else: @@ -441,7 +437,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): for item in reversed(self): - item.keep = getattr(self, 'keep', None) + item.keep = getattr(self, 'keep', False) item.__exit__(exc_type, exc_val, exc_tb) # diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index dbcf89d8645..ea425127c41 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -277,3 +277,38 @@ def test_restage(self): self.check_chdir_to_source(stage, stage_name) self.assertFalse('foobar' in os.listdir(stage.source_path)) self.check_destroy(stage, stage_name) + + + def test_no_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=False) as stage: + pass + self.check_destroy(stage, stage_name) + + + def test_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=True) as stage: + pass + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + + + def test_no_keep_with_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=False) as stage: + raise Exception() + + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. + + + def test_keep_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=True) as stage: + raise Exception() + + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. From a384ad5b1270140d71110e46d39144a0f0e9081e Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 9 Mar 2016 16:11:33 -0800 Subject: [PATCH 231/339] Fix problem with pure integer arguments in preferred versions list (e.g, 2 instead of 2.7.3) --- lib/spack/spack/config.py | 5 +++-- lib/spack/spack/preferred_packages.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 807a898644d..95a988f7ffe 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -214,7 +214,8 @@ 'version': { 'type' : 'array', 'default' : [], - 'items' : { 'type' : 'string' } }, #version strings + 'items' : { 'anyOf' : [ { 'type' : 'string' }, + { 'type' : 'number'}]}}, #version strings 'compiler': { 'type' : 'array', 'default' : [], @@ -573,7 +574,7 @@ def __init__(self, validation_error, data): # Try to get line number from erroneous instance and its parent instance_mark = getattr(validation_error.instance, '_start_mark', None) parent_mark = getattr(validation_error.parent, '_start_mark', None) - path = getattr(validation_error, 'path', None) + path = [str(s) for s in getattr(validation_error, 'path', None)] # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 2b0ba791b6b..eaea016a85a 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -45,7 +45,7 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): order = order.get(second_key, {}) if not order: continue - return [s.strip() for s in order] + return [str(s).strip() for s in order] return [] @@ -98,11 +98,11 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon b_index = None reverse = -1 if reverse_natural_compare else 1 for i, cspec in enumerate(specs): - if a_index == None and cspec.satisfies(a): + if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)): a_index = i if b_index: break - if b_index == None and cspec.satisfies(b): + if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)): b_index = i if a_index: break From 1f06dd40f7e65252568da23e9758bf5af02833eb Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Wed, 9 Mar 2016 16:11:53 -0800 Subject: [PATCH 232/339] Update documentation for new packages.yaml config format. --- lib/spack/docs/packaging_guide.rst | 41 +++++++++---------- lib/spack/docs/site_configuration.rst | 59 +++++++++++++-------------- 2 files changed, 48 insertions(+), 52 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index f368d0a4fa8..ef9fd89b62f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1561,50 +1561,49 @@ be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -Spack's ``preferred`` configuration can be used to set defaults for sites or users. -Spack uses this configuration to make decisions about which compilers, package -versions, depends_on, and variants it should prefer during concretization. - -The preferred configuration can be controlled by editing the -``~/.spack/preferred.yaml`` file for user configuations, or the +Spack can be configurated to prefer certain compilers, package +versions, depends_on, and variants during concretization. +The preferred configuration can be controlled via the +``~/.spack/packages.yaml`` file for user configuations, or the +``etc/spack/packages.yaml`` site configuration. -Here's an example preferred.yaml file: +Here's an example packages.yaml file that sets preferred packages: .. code-block:: sh - preferred: + packages: dyninst: - compiler: gcc@4.9 + compiler: [gcc@4.9] variants: +debug gperftools: - version: 2.2, 2.4, 2.3 + version: [2.2, 2.4, 2.3] all: - compiler: gcc@4.4.7, gcc@4.6:, intel, clang, pgi - providers: - mpi: mvapich, mpich, openmpi + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] + At a high level, this example is specifying how packages should be concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which perfers gcc 4.9). +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each preferred.yaml file begin with the string ``preferred:`` and -each subsequent entry is indented underneath it. The next layer contains -package names or the special string ``all`` (which applies to -every package). Underneath each package name is +Each packages.yaml file begin with the string ``packages:`` and +package names are specified on the next level. The special string ``all`` +applies settings to each package. Underneath each package name is one or more components: ``compiler``, ``variants``, ``version``, or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being prefered over -latter entries. +later entries. Sometimes a package installation may have constraints that forbid the first concretization rule, in which case Spack will use the first legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or latter, then Spack will install version 2.4 +requests gperftools 2.3 or later, then Spack will install version 2.4 as the 2.4 version of gperftools is preferred over 2.3. An explicit concretization rule in the preferred section will always @@ -1612,7 +1611,7 @@ take preference over unlisted concretizations. In the above example, xlc isn't listed in the compiler list. Every listed compiler from gcc to pgi will thus be preferred over the xlc compiler. -The syntax for the ``providers`` section differs slightly from other +The syntax for the ``provider`` section differs slightly from other concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index a7211a9d95b..ebf04371067 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -56,44 +56,43 @@ directory is. External Packages ~~~~~~~~~~~~~~~~~~~~~ -It's possible for Spack to use certain externally-installed -packages rather than always rebuilding packages. This may be desirable +Spack can be configured to use externally-installed +packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI that should be used instead of Spack building its own MPI. External packages are configured through the ``packages.yaml`` file found in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` -directory. Here's an example of an external configuration:: +directory. Here's an example of an external configuration: .. code-block:: yaml - packages: - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: - path: /opt/openmpi-1.4.3 - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: - path: /opt/openmpi-1.4.3-debug - - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: - path: /opt/openmpi-1.6.5-intel + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with OpenMPI. If Spack is asked to build a package that uses one of these MPIs as a -dependency, it link the package to the pre-installed OpenMPI in -the given directory. +dependency, it will use the the pre-installed OpenMPI in +the given directory. This example also specifies that Spack should never +build its own OpenMPI via the ``nobuild: True`` option. -Each ``packages.yaml`` should begin with a ``packages:`` token, followed -by a list of package specs. Specs in the ``packages.yaml`` have at most -one ``path`` tag, which specifies the top-level directory where the -spec is installed. - -Each spec should be as well-defined as reasonably possible. If a +Each ``packages.yaml`` begins with a ``packages:`` token, followed +by a list of package names. To specify externals, add a ``paths`` +token under the package name, which lists externals in a +``spec : /path`` format. Each spec should be as +well-defined as reasonably possible. If a package lacks a spec component, such as missing a compiler or package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly. -All package versions and compilers listed in ``packages.yaml`` should +Each package version and compilers listed in an external should have entries in Spack's packages and compiler configuration, even -the package and compiler may not actually be used. +though the package and compiler may not every be built. The packages configuration can tell Spack to use an external location for certain package versions, but it does not restrict Spack to using @@ -103,27 +102,25 @@ rather than continue using the pre-installed OpenMPI versions. To prevent this, the ``packages.yaml`` configuration also allows packages to be flagged as non-buildable. The previous example could be modified to -be:: +be: .. code-block:: yaml packages: - - openmpi: - nobuild: True - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: - path: /opt/openmpi-1.4.3 - - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: - path: /opt/openmpi-1.4.3-debug - - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: - path: /opt/openmpi-1.6.5-intel + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + nobuild: True The addition of the ``nobuild`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``path``, ``nobuild`` is specified as a property under -a spec and will prevent building of anything that satisfies that spec. +a package name. The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid versions of packages that may be +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. From 23cbc2b1d9dc5816c63f330640105c5435ccdc22 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Tue, 8 Mar 2016 09:44:21 -0700 Subject: [PATCH 233/339] + Provide download/build instructions for qt/5.4.2. - This version provides updates to provided cmake scripts that are required for building cmake-gui. + Provide download/build instructions for version 3.5.0. - When building the +qt variant, add a validate function to ensure that qt-5.4.0 is not used (this version of qt has errors related to cmake). --- .../repos/builtin/packages/cmake/package.py | 21 +++++++++++++++++-- .../repos/builtin/packages/qt/package.py | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index f39a6812843..806c37a68cc 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -30,6 +30,7 @@ class Cmake(Package): homepage = 'https://www.cmake.org' url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') @@ -49,8 +50,25 @@ def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) - def install(self, spec, prefix): + def validate(self, spec): + """ + Checks if incompatible versions of qt were specified + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + + print spec + + if '+qt' in spec and spec.satisfies('^qt@5.4.0'): + msg = 'qt-5.4.0 has broken CMake modules.' + raise RuntimeError(msg) + + def install(self, spec, prefix): + # Consistency check + self.validate(spec) + + # configure, build, install: options = ['--prefix=%s' % prefix] options.append('--parallel=%s' % str(make_jobs)) @@ -65,6 +83,5 @@ def install(self, spec, prefix): options.append('-DCMAKE_USE_OPENSSL=ON') configure(*options) - make() make('install') diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..ef5f05601f8 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -8,6 +8,9 @@ class Qt(Package): list_url = 'http://download.qt-project.org/official_releases/qt/' list_depth = 2 + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') From f56939c16c36ee946f40571bdc330b432af93cce Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 17:01:01 -0800 Subject: [PATCH 234/339] Add unit test for sbang patching. --- lib/spack/spack/test/__init__.py | 13 ++--- lib/spack/spack/test/sbang.py | 93 ++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 lib/spack/spack/test/sbang.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 4b9a361d4bf..d5d8b647650 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -65,7 +65,8 @@ 'lock', 'database', 'namespace_trie', - 'yaml'] + 'yaml', + 'sbang'] def list_tests(): @@ -87,20 +88,20 @@ def run(names, outputDir, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - + tally = Tally() for test in names: module = 'spack.test.' + test print module - + tty.msg("Running test: %s" % test) - + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name] - + if outputDir: xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) - runOpts += ["--with-xunit", + runOpts += ["--with-xunit", "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] result = nose.run(argv=argv, addplugins=[tally]) diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py new file mode 100644 index 00000000000..825bc4be982 --- /dev/null +++ b/lib/spack/spack/test/sbang.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""\ +Test that Spack's shebang filtering works correctly. +""" +import os +import unittest +import tempfile +import shutil + +from llnl.util.filesystem import * +from spack.hooks.sbang import filter_shebangs_in_directory +import spack + +short_line = "#!/this/is/short/bin/bash\n" +long_line = "#!/this/" + ('x' * 200) + "/is/long\n" +sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root +last_line = "last!\n" + +class SbangTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + # make sure we can ignore non-files + directory = os.path.join(self.tempdir, 'dir') + mkdirp(directory) + + # Script with short shebang + self.short_shebang = os.path.join(self.tempdir, 'short') + with open(self.short_shebang, 'w') as f: + f.write(short_line) + f.write(last_line) + + # Script with long shebang + self.long_shebang = os.path.join(self.tempdir, 'long') + with open(self.long_shebang, 'w') as f: + f.write(long_line) + f.write(last_line) + + # Script already using sbang. + self.has_shebang = os.path.join(self.tempdir, 'shebang') + with open(self.has_shebang, 'w') as f: + f.write(sbang_line) + f.write(long_line) + f.write(last_line) + + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + + + def test_shebang_handling(self): + filter_shebangs_in_directory(self.tempdir) + + # Make sure this is untouched + with open(self.short_shebang, 'r') as f: + self.assertEqual(f.readline(), short_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this got patched. + with open(self.long_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this is untouched + with open(self.has_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) From 52cdcdde76b272ba4905a23642323d116b00a7e7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Mar 2016 17:13:04 -0800 Subject: [PATCH 235/339] Fix #525: sbang handles symlinks & directories properly. --- lib/spack/spack/hooks/sbang.py | 43 +++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 3390ecea295..d78adb576e2 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -35,7 +35,7 @@ shebang_limit = 127 def shebang_too_long(path): - """Detects whether an file has a shebang line that is too long.""" + """Detects whether a file has a shebang line that is too long.""" with open(path, 'r') as script: bytes = script.read(2) if bytes != '#!': @@ -47,14 +47,21 @@ def shebang_too_long(path): def filter_shebang(path): """Adds a second shebang line, using sbang, at the beginning of a file.""" + with open(path, 'r') as original_file: + original = original_file.read() + + # This line will be prepended to file + new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root + + # Skip files that are already using sbang. + if original.startswith(new_sbang_line): + return + backup = path + ".shebang.bak" os.rename(path, backup) - with open(backup, 'r') as bak_file: - original = bak_file.read() - with open(path, 'w') as new_file: - new_file.write('#!/bin/bash %s/bin/sbang\n' % spack.spack_root) + new_file.write(new_sbang_line) new_file.write(original) copy_mode(backup, path) @@ -63,15 +70,29 @@ def filter_shebang(path): tty.warn("Patched overly long shebang in %s" % path) +def filter_shebangs_in_directory(directory): + for file in os.listdir(directory): + path = os.path.join(directory, file) + + # only handle files + if not os.path.isfile(path): + continue + + # only handle links that resolve within THIS package's prefix. + if os.path.islink(path): + real_path = os.path.realpath(path) + if not real_path.startswith(directory + os.sep): + continue + + # test the file for a long shebang, and filter + if shebang_too_long(path): + filter_shebang(path) + + def post_install(pkg): """This hook edits scripts so that they call /bin/bash $spack_prefix/bin/sbang instead of something longer than the shebang limit.""" if not os.path.isdir(pkg.prefix.bin): return - - for file in os.listdir(pkg.prefix.bin): - path = os.path.join(pkg.prefix.bin, file) - if shebang_too_long(path): - filter_shebang(path) - + filter_shebangs_in_directory(pkg.prefix.bin) From 267e83d8a70324d9b8a70a86085853dd762c7f93 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 21:12:11 -0500 Subject: [PATCH 236/339] Added emacs package. --- .../repos/builtin/packages/emacs/package.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 var/spack/repos/builtin/packages/emacs/package.py diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py new file mode 100644 index 00000000000..e496ffc3d1f --- /dev/null +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -0,0 +1,36 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install emacs +# +# You can always get back here to change things with: +# +# spack edit emacs +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Emacs(Package): + """FIXME: put a proper description of your package here.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" + + version('24.5', 'd74b597503a68105e61b5b9f6d065b44') + + # FIXME: Add dependencies if this package requires them. + depends_on('ncurses') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") From b701aa10d4b40eda3d5ea3f0c96b5aed7493c7a7 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Wed, 9 Mar 2016 21:21:25 -0500 Subject: [PATCH 237/339] Fixed up --- .../repos/builtin/packages/emacs/package.py | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index e496ffc3d1f..09eb05d5a76 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -1,36 +1,19 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install emacs -# -# You can always get back here to change things with: -# -# spack edit emacs -# -# See the spack documentation for more information on building -# packages. -# from spack import * class Emacs(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.example.com" + """The Emacs programmable text editor.""" + homepage = "https://www.gnu.org/software/emacs" url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" version('24.5', 'd74b597503a68105e61b5b9f6d065b44') - # FIXME: Add dependencies if this package requires them. depends_on('ncurses') + # Emacs also depends on: + # GTK or other widget library + # libtiff, png, etc. + # For now, we assume the system provides all that stuff. def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure('--prefix=%s' % prefix) - - # FIXME: Add logic to build and install here make() make("install") From b0377da771d9154956a408a59e97000049d7c2fb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 12:21:04 -0800 Subject: [PATCH 238/339] update mirror config documentation. - mirrors.yaml uses Spack's OrderedDict rather than lists. --- lib/spack/docs/mirrors.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 7581a0e9ede..b20fedb55f1 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later. ``spack mirror list`` ---------------------------- -If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: +To see all the mirrors Spack knows about, run ``spack mirror list``:: $ spack mirror list local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror ``spack mirror remove`` ---------------------------- -And, if you want to remove a mirror, just remove it by name:: +To remove a mirror by name:: $ spack mirror remove local_filesystem $ spack mirror list @@ -205,11 +205,11 @@ And, if you want to remove a mirror, just remove it by name:: Mirror precedence ---------------------------- -Adding a mirror really just adds a section in ``~/.spack/mirrors.yaml``:: +Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: mirrors: - - local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 - - remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 + remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will From 0244d794cd68efd68edd6a797dd1db158aca87b6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 31 Jan 2016 13:28:12 -0800 Subject: [PATCH 239/339] remove unnecessary import --- lib/spack/spack/stage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 4703a3aae6a..5354135e6a3 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,7 +26,6 @@ import errno import shutil import tempfile -import sys from urlparse import urljoin import llnl.util.tty as tty From b0572a546242ef1c570f5dd3c9c6336bc1d55607 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Feb 2016 11:27:39 -0700 Subject: [PATCH 240/339] Minor tweaks to abi code. --- lib/spack/spack/abi.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index f0a997703cd..7e565bcbf9e 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -69,11 +69,11 @@ def _gcc_get_libstdcxx_version(self, version): if not libpath: return None return os.path.basename(libpath) - - + + @memoized def _gcc_compiler_compare(self, pversion, cversion): - """Returns true iff the gcc version pversion and cversion + """Returns true iff the gcc version pversion and cversion are ABI compatible.""" plib = self._gcc_get_libstdcxx_version(pversion) clib = self._gcc_get_libstdcxx_version(cversion) @@ -86,43 +86,43 @@ def _intel_compiler_compare(self, pversion, cversion): """Returns true iff the intel version pversion and cversion are ABI compatible""" - #Test major and minor versions. Ignore build version. + # Test major and minor versions. Ignore build version. if (len(pversion.version) < 2 or len(cversion.version) < 2): return False - return (pversion.version[0] == cversion.version[0]) and \ - (pversion.version[1] == cversion.version[1]) - - + return pversion.version[:2] == cversion.version[:2] + + def compiler_compatible(self, parent, child, **kwargs): """Returns true iff the compilers for parent and child specs are ABI compatible""" if not parent.compiler or not child.compiler: return True - + if parent.compiler.name != child.compiler.name: - #Different compiler families are assumed ABI incompatible + # Different compiler families are assumed ABI incompatible return False - + if kwargs.get('loose', False): return True + # TODO: Can we move the specialized ABI matching stuff + # TODO: into compiler classes? for pversion in parent.compiler.versions: for cversion in child.compiler.versions: - #For a few compilers use specialized comparisons. Otherwise + # For a few compilers use specialized comparisons. Otherwise # match on version match. if pversion.satisfies(cversion): return True - elif parent.compiler.name == "gcc" and \ - self._gcc_compiler_compare(pversion, cversion): + elif (parent.compiler.name == "gcc" and + self._gcc_compiler_compare(pversion, cversion)): return True - elif parent.compiler.name == "intel" and \ - self._intel_compiler_compare(pversion, cversion): + elif (parent.compiler.name == "intel" and + self._intel_compiler_compare(pversion, cversion)): return True return False - + def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) return self.architecture_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) - From 048c406f49a3c7a30008268590ab57b74ea60b6b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Feb 2016 11:32:26 -0700 Subject: [PATCH 241/339] Remove vestigial variants in directory name. --- lib/spack/spack/directory_layout.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index b94468faf01..39ee4e203d6 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -198,14 +198,10 @@ def hidden_file_paths(self): def relative_path_for_spec(self, spec): _check_concrete(spec) - + if spec.external: return spec.external - enabled_variants = ( - '-' + v.name for v in spec.variants.values() - if v.enabled) - dir_name = "%s-%s-%s" % ( spec.name, spec.version, From 1fe196f95cc26cac73abe64752ff67b150f4d50a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 2 Mar 2016 22:38:21 -0800 Subject: [PATCH 242/339] whitespace and formatting --- lib/spack/spack/preferred_packages.py | 38 +++++++++---------- lib/spack/spack/spec.py | 8 ++-- .../repos/builtin/packages/python/package.py | 5 ++- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index eaea016a85a..4ff0f18b31d 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -33,8 +33,8 @@ def __init__(self): self.preferred = spack.config.get_config('packages') self._spec_for_pkgname_cache = {} - #Given a package name, sort component (e.g, version, compiler, ...), and - # a second_key (used by providers), return the list + # Given a package name, sort component (e.g, version, compiler, ...), and + # a second_key (used by providers), return the list def _order_for_package(self, pkgname, component, second_key, test_all=True): pkglist = [pkgname] if test_all: @@ -47,10 +47,10 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): continue return [str(s).strip() for s in order] return [] - - # A generic sorting function. Given a package name and sort - # component, return less-than-0, 0, or greater-than-0 if + + # A generic sorting function. Given a package name and sort + # component, return less-than-0, 0, or greater-than-0 if # a is respectively less-than, equal to, or greater than b. def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): if a is None: @@ -76,7 +76,7 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, cmp_a = orderlist.index(str(a)) cmp_b = orderlist.index(str(b)) reverse = 1 - + if cmp_a < cmp_b: return -1 * reverse elif cmp_a > cmp_b: @@ -87,7 +87,7 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, # A sorting function for specs. Similar to component_compare, but # a and b are considered to match entries in the sorting list if they - # satisfy the list component. + # satisfy the list component. def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): if not a or not a.concrete: return -1 @@ -121,7 +121,7 @@ def _spec_for_pkgname(self, pkgname, component, second_key): key = (pkgname, component, second_key) if not key in self._spec_for_pkgname_cache: pkglist = self._order_for_package(pkgname, component, second_key) - if not pkglist: + if not pkglist: if component in self._default_order: pkglist = self._default_order[component] if component == 'compiler': @@ -132,9 +132,9 @@ def _spec_for_pkgname(self, pkgname, component, second_key): self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] return self._spec_for_pkgname_cache[key] - + def provider_compare(self, pkgname, provider_str, a, b): - """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or + """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or greater-than b. A and b are possible implementations of provider_str. One provider is less-than another if it is preferred over the other. For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if @@ -148,28 +148,28 @@ def spec_has_preferred_provider(self, pkgname, provider_str): def version_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if version a of pkgname is - respecively less-than, equal-to, or greater-than version b of pkgname. + """Return less-than-0, 0, or greater than 0 if version a of pkgname is + respecively less-than, equal-to, or greater-than version b of pkgname. One version is less-than another if it is preferred over the other.""" return self._spec_compare(pkgname, 'version', a, b, True, None) - + def variant_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if variant a of pkgname is - respecively less-than, equal-to, or greater-than variant b of pkgname. + """Return less-than-0, 0, or greater than 0 if variant a of pkgname is + respecively less-than, equal-to, or greater-than variant b of pkgname. One variant is less-than another if it is preferred over the other.""" return self._component_compare(pkgname, 'variant', a, b, False, None) def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is + respecively less-than, equal-to, or greater-than architecture b of pkgname. One architecture is less-than another if it is preferred over the other.""" return self._component_compare(pkgname, 'architecture', a, b, False, None) def compiler_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is - respecively less-than, equal-to, or greater-than compiler b of pkgname. + """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is + respecively less-than, equal-to, or greater-than compiler b of pkgname. One compiler is less-than another if it is preferred over the other.""" return self._spec_compare(pkgname, 'compiler', a, b, False, None) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6f55065f010..b8c0d0ef9c5 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -428,7 +428,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): for dep in dep_like: spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - + # # Private routines here are called by the parser when building a spec. @@ -1410,7 +1410,7 @@ def _dup(self, other, **kwargs): self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external) - + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() @@ -1585,7 +1585,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): $@ Version with '@' prefix $% Compiler with '%' prefix $%@ Compiler with '%' prefix & compiler version with '@' prefix - $+ Options + $+ Options $= Architecture with '=' prefix $# 7-char prefix of DAG hash with '-' prefix $$ $ @@ -1738,7 +1738,7 @@ def __cmp__(self, other): #Package name sort order is not configurable, always goes alphabetical if self.name != other.name: return cmp(self.name, other.name) - + #Package version is second in compare order pkgname = self.name if self.versions != other.versions: diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 58d401244e3..dd240d1ea0a 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -34,8 +34,9 @@ def install(self, spec, prefix): env['PYTHONHOME'] = prefix env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' - # Rest of install is pretty standard except setup.py needs to be able to read the CPPFLAGS - # and LDFLAGS as it scans for the library and headers to build + # Rest of install is pretty standard except setup.py needs to + # be able to read the CPPFLAGS and LDFLAGS as it scans for the + # library and headers to build configure_args= [ "--prefix=%s" % prefix, "--with-threads", From 82b7067fdfc3f2fb90cff9014ab5379e334b40fd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 3 Mar 2016 00:44:00 -0800 Subject: [PATCH 243/339] Refactored external packages slightly. - Move `Spec.__cmp__` out of spec, into concretize as `cmp_specs`. - `Spec.__cmp__` was never called (except explicitly) due to rich comparison operators from `key_ordering` - Refactor `_find_other_spec` to free function `find_spec`. Add a test for it to make sure it works. --- lib/spack/spack/concretize.py | 142 +++++++++++++++++--------- lib/spack/spack/preferred_packages.py | 2 +- lib/spack/spack/spec.py | 34 ------ lib/spack/spack/test/concretize.py | 64 ++++++++++++ 4 files changed, 158 insertions(+), 84 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8da7011b538..bad67c34e3f 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -50,34 +50,17 @@ class DefaultConcretizer(object): default concretization strategies, or you can override all of them. """ - def _find_other_spec(self, spec, condition): - """Searches the dag from spec in an intelligent order and looks - for a spec that matches a condition""" - dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) - found = next((x for x in dagiter if x is not spec and condition(x)), None) - if found: - return found - dagiter = chain(spec.traverse(direction='parents'), spec.traverse(direction='children')) - searched = list(dagiter) - found = next((x for x in spec.root.traverse() if x not in searched and x is not spec and condition(x)), None) - if found: - return found - if condition(spec): - return spec - return None - - def _valid_virtuals_and_externals(self, spec): """Returns a list of spec/external-path pairs for both virtuals and externals - that can concretize this spec.""" + that can concretize this spec.""" # Get a list of candidate packages that could satisfy this spec packages = [] if spec.virtual: providers = spack.repo.providers_for(spec) if not providers: raise UnsatisfiableProviderSpecError(providers[0], spec) - spec_w_preferred_providers = self._find_other_spec(spec, \ - lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + spec_w_preferred_providers = find_spec( + spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) if not spec_w_preferred_providers: spec_w_preferred_providers = spec provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) @@ -101,15 +84,15 @@ def _valid_virtuals_and_externals(self, spec): raise NoBuildError(spec) def cmp_externals(a, b): - result = a[0].__cmp__(b[0]) - if result != 0: return result + result = cmp_specs(a[0], b[0]) + if result != 0: + return result if not a[1] and b[1]: return 1 if not b[1] and a[1]: return -1 - return a[1].__cmp__(b[1]) + return cmp_specs(a[1], b[1]) - #result = sorted(result, cmp=lambda a,b: a[0].__cmp__(b[0])) result = sorted(result, cmp=cmp_externals) return result @@ -121,27 +104,27 @@ def concretize_virtual_and_external(self, spec): if not candidates: return False - #Find the nearest spec in the dag that has a compiler. We'll use that + # Find the nearest spec in the dag that has a compiler. We'll use that # spec to test compiler compatibility. - other_spec = self._find_other_spec(spec, lambda(x): x.compiler) + other_spec = find_spec(spec, lambda(x): x.compiler) if not other_spec: other_spec = spec.root - #Choose an ABI-compatible candidate, or the first match otherwise. + # Choose an ABI-compatible candidate, or the first match otherwise. candidate = None if other_spec: candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) if not candidate: - #Try a looser ABI matching + # Try a looser ABI matching candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) if not candidate: - #No ABI matches. Pick the top choice based on the orignal preferences. + # No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] candidate_spec = candidate[0] external = candidate[1] changed = False - #If we're external then trim the dependencies + # If we're external then trim the dependencies if external: if (spec.dependencies): changed = True @@ -150,26 +133,26 @@ def concretize_virtual_and_external(self, spec): def fequal(candidate_field, spec_field): return (not candidate_field) or (candidate_field == spec_field) - if fequal(candidate_spec.name, spec.name) and \ - fequal(candidate_spec.versions, spec.versions) and \ - fequal(candidate_spec.compiler, spec.compiler) and \ - fequal(candidate_spec.architecture, spec.architecture) and \ - fequal(candidate_spec.dependencies, spec.dependencies) and \ - fequal(candidate_spec.variants, spec.variants) and \ - fequal(external, spec.external): + if (fequal(candidate_spec.name, spec.name) and + fequal(candidate_spec.versions, spec.versions) and + fequal(candidate_spec.compiler, spec.compiler) and + fequal(candidate_spec.architecture, spec.architecture) and + fequal(candidate_spec.dependencies, spec.dependencies) and + fequal(candidate_spec.variants, spec.variants) and + fequal(external, spec.external)): return changed - - #Refine this spec to the candidate. + + # Refine this spec to the candidate. if spec.virtual: spec._replace_with(candidate_spec) changed = True if spec._dup(candidate_spec, deps=False, cleardeps=False): changed = True - spec.external = external + spec.external = external return changed - - + + def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take the preferred version from spackconfig, and default to the package's @@ -263,7 +246,7 @@ def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not - concrete, then used the preferred compiler as specified in + concrete, then used the preferred compiler as specified in spackconfig. Intuition: Use the spackconfig default if no package that depends on @@ -272,37 +255,99 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() - + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return False #Find the another spec that has a compiler, or the root if none do - other_spec = self._find_other_spec(spec, lambda(x) : x.compiler) + other_spec = find_spec(spec, lambda(x) : x.compiler) if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler assert(other_spec) - + # Check if the compiler is already fully specified if other_compiler in all_compilers: spec.compiler = other_compiler.copy() return True - + # Filter the compilers into a sorted list based on the compiler_order from spackconfig compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: raise UnavailableCompilerVersionError(other_compiler) - + # copy concrete version into other_compiler spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. +def find_spec(spec, condition): + """Searches the dag from spec in an intelligent order and looks + for a spec that matches a condition""" + # First search parents, then search children + dagiter = chain(spec.traverse(direction='parents', root=False), + spec.traverse(direction='children', root=False)) + visited = set() + for relative in dagiter: + if condition(relative): + return relative + visited.add(id(relative)) + + # Then search all other relatives in the DAG *except* spec + for relative in spec.root.traverse(): + if relative is spec: continue + if id(relative) in visited: continue + if condition(relative): + return relative + + # Finally search spec itself. + if condition(spec): + return spec + + return None # Nohting matched the condition. + + +def cmp_specs(lhs, rhs): + # Package name sort order is not configurable, always goes alphabetical + if lhs.name != rhs.name: + return cmp(lhs.name, rhs.name) + + # Package version is second in compare order + pkgname = lhs.name + if lhs.versions != rhs.versions: + return spack.pkgsort.version_compare( + pkgname, lhs.versions, rhs.versions) + + # Compiler is third + if lhs.compiler != rhs.compiler: + return spack.pkgsort.compiler_compare( + pkgname, lhs.compiler, rhs.compiler) + + # Variants + if lhs.variants != rhs.variants: + return spack.pkgsort.variant_compare( + pkgname, lhs.variants, rhs.variants) + + # Architecture + if lhs.architecture != rhs.architecture: + return spack.pkgsort.architecture_compare( + pkgname, lhs.architecture, rhs.architecture) + + # Dependency is not configurable + lhash, rhash = hash(lhs), hash(rhs) + if lhash != rhash: + return -1 if lhash < rhash else 1 + + # Equal specs + return 0 + + + class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" @@ -326,4 +371,3 @@ class NoBuildError(spack.error.SpackError): def __init__(self, spec): super(NoBuildError, self).__init__( "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) - diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 4ff0f18b31d..9d219a1a6ef 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -27,7 +27,7 @@ from spack.version import * class PreferredPackages(object): - _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, #Arbitrary, but consistent + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, # Arbitrary, but consistent def __init__(self): self.preferred = spack.config.get_config('packages') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b8c0d0ef9c5..c045e803655 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1734,40 +1734,6 @@ def dep_string(self): return ''.join("^" + dep.format() for dep in self.sorted_deps()) - def __cmp__(self, other): - #Package name sort order is not configurable, always goes alphabetical - if self.name != other.name: - return cmp(self.name, other.name) - - #Package version is second in compare order - pkgname = self.name - if self.versions != other.versions: - return spack.pkgsort.version_compare(pkgname, - self.versions, other.versions) - - #Compiler is third - if self.compiler != other.compiler: - return spack.pkgsort.compiler_compare(pkgname, - self.compiler, other.compiler) - - #Variants - if self.variants != other.variants: - return spack.pkgsort.variant_compare(pkgname, - self.variants, other.variants) - - #Architecture - if self.architecture != other.architecture: - return spack.pkgsort.architecture_compare(pkgname, - self.architecture, other.architecture) - - #Dependency is not configurable - if self.dag_hash() != other.dag_hash(): - return -1 if self.dag_hash() < other.dag_hash() else 1 - - #Equal specs - return 0 - - def __str__(self): return self.format() + self.dep_string() diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 820c5d84a89..07828d8ea6c 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -24,6 +24,7 @@ ############################################################################## import spack from spack.spec import Spec, CompilerSpec +from spack.concretize import find_spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): @@ -218,3 +219,66 @@ def test_external_and_virtual(self): self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc') self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) self.assertTrue(spec['stuff'].compiler.satisfies('gcc')) + + + def test_find_spec_parents(self): + """Tests the spec finding logic used by concretization. """ + s = Spec('a +foo', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + + self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_children(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name) + s = Spec('a', + Spec('b +foo', + Spec('c +foo'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_sibling(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name) + self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name) + + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e', + Spec('f +foo'))) + self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_self(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_none(self): + s = Spec('a', + Spec('b', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) + From c4fddcc6e977eb7040c48581eb90f1a441cf3e71 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 03:15:49 -0800 Subject: [PATCH 244/339] Add 'provders' back into packages.yaml schema --- lib/spack/spack/config.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 95a988f7ffe..3a785fe6924 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -224,6 +224,15 @@ 'type': 'boolean', 'default': False, }, + 'providers': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' },},},}, 'paths': { 'type' : 'object', 'default' : {}, @@ -534,11 +543,11 @@ def spec_externals(spec): allpkgs = get_config('packages') name = spec.name spec_locations = [] - + pkg_paths = allpkgs.get(name, {}).get('paths', None) if not pkg_paths: return [] - + for pkg,path in pkg_paths.iteritems(): if not spec.satisfies(pkg): continue From 7ae6c62ddb0271d9e2364cf4249adbc7abbc0b31 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 10 Mar 2016 14:08:43 +0100 Subject: [PATCH 245/339] Fixed shell quoting error --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 6b6073db430..2888c28c489 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -154,7 +154,7 @@ fi input_command="$@" if [ "$mode" == vcheck ] ; then - exec "${input_command}" + exec ${command} ${input_command} fi # From 78ef0618bc6d60f50edc6bb83b5cb1a5a48ac22b Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 10 Mar 2016 14:35:09 +0100 Subject: [PATCH 246/339] cc : converted indents to spaces --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 2888c28c489..3e429fdf220 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -119,7 +119,7 @@ if [ -z "$mode" ]; then if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then mode=vcheck break - fi + fi done fi From 30adc4c9b8cdbe099ebe4ef9bec4b2594cda7b61 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 10 Mar 2016 14:59:31 +0100 Subject: [PATCH 247/339] cc : handle spaces in folders? --- lib/spack/env/cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 3e429fdf220..4a3e6eddc94 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -154,7 +154,7 @@ fi input_command="$@" if [ "$mode" == vcheck ] ; then - exec ${command} ${input_command} + exec ${command} "$@" fi # From 2bbf42b49d874adad2d9f53e1c8f6225df80415b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 16:18:11 -0800 Subject: [PATCH 248/339] Indentation change. --- lib/spack/docs/site_configuration.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 76fe8fdd7d2..7ae541d4a39 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -56,7 +56,7 @@ directory is. External Packages ~~~~~~~~~~~~~~~~~~~~~ -Spack can be configured to use externally-installed +Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI that should be used instead of Spack building its own MPI. @@ -76,7 +76,7 @@ directory. Here's an example of an external configuration: This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. -If Spack is asked to build a package that uses one of these MPIs as a +If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in the given directory. This example also specifies that Spack should never build its own OpenMPI via the ``nobuild: True`` option. @@ -85,12 +85,12 @@ Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` token under the package name, which lists externals in a ``spec : /path`` format. Each spec should be as -well-defined as reasonably possible. If a -package lacks a spec component, such as missing a compiler or -package version, then Spack will guess the missing component based +well-defined as reasonably possible. If a +package lacks a spec component, such as missing a compiler or +package version, then Spack will guess the missing component based on its most-favored packages, and it may guess incorrectly. -Each package version and compilers listed in an external should +Each package version and compilers listed in an external should have entries in Spack's packages and compiler configuration, even though the package and compiler may not every be built. @@ -111,18 +111,18 @@ be: paths: openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel - nobuild: True + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + nobuild: True The addition of the ``nobuild`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built OpenMPI. Similar to ``paths``, ``nobuild`` is specified as a property under a package name. -The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid packages that may be +The ``nobuild`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. - + Profiling ~~~~~~~~~~~~~~~~~~~~~ From 4693af0736910244abcb193a65041314447f0da6 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Thu, 10 Mar 2016 16:23:35 -0800 Subject: [PATCH 249/339] Fix type error that was causing mis-ordering of compiler versions --- lib/spack/spack/preferred_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 9d219a1a6ef..4d8526c75fa 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -27,7 +27,7 @@ from spack.version import * class PreferredPackages(object): - _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] }, # Arbitrary, but consistent + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent def __init__(self): self.preferred = spack.config.get_config('packages') From ac88cab68ffa16d4ff448de5f186d746cb36b40a Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Thu, 10 Mar 2016 17:00:27 -0800 Subject: [PATCH 250/339] Fix issue with preferred satisfies not being respected --- lib/spack/spack/concretize.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index bad67c34e3f..2268084e568 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -70,20 +70,23 @@ def _valid_virtuals_and_externals(self, spec): # For each candidate package, if it has externals add those to the candidates # if it's a nobuild, then only add the externals. - result = [] + candidates = [] all_compilers = spack.compilers.all_compilers() for pkg in packages: externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + candidates.append((pkg, None)) for ext in externals: if ext[0].satisfies(spec): - result.append(ext) - if not result: + candidates.append(ext) + if not candidates: raise NoBuildError(spec) def cmp_externals(a, b): + if a[0].name != b[0].name: + #We're choosing between different providers. Maintain order from above sort + return candidates.index(a) - candidates.index(b) result = cmp_specs(a[0], b[0]) if result != 0: return result @@ -91,10 +94,10 @@ def cmp_externals(a, b): return 1 if not b[1] and a[1]: return -1 - return cmp_specs(a[1], b[1]) + return cmp(a[1], b[1]) - result = sorted(result, cmp=cmp_externals) - return result + candidates = sorted(candidates, cmp=cmp_externals) + return candidates def concretize_virtual_and_external(self, spec): From 1d70b590fc4579c4e6b4bd592bc58eabf330fd9b Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 11 Mar 2016 13:20:57 +0100 Subject: [PATCH 251/339] build_environment : fixed minor spelling errors and a few style issues --- lib/spack/spack/build_environment.py | 37 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a3..392ba7ea4d5 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -3,7 +3,7 @@ build environment. All of this is set up by package.py just before install() is called. -There are two parts to the bulid environment: +There are two parts to the build environment: 1. Python build environment (i.e. install() method) @@ -13,7 +13,7 @@ the package's module scope. Ths allows package writers to call them all directly in Package.install() without writing 'self.' everywhere. No, this isn't Pythonic. Yes, it makes the code more - readable and more like the shell script from whcih someone is + readable and more like the shell script from which someone is likely porting. 2. Build execution environment @@ -27,17 +27,16 @@ Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ -import os -import sys -import shutil import multiprocessing +import os import platform -from llnl.util.filesystem import * +import shutil +import sys import spack -import spack.compilers as compilers -from spack.util.executable import Executable, which +from llnl.util.filesystem import * from spack.util.environment import * +from spack.util.executable import Executable, which # # This can be set by the user to globally disable parallel builds. @@ -107,18 +106,19 @@ def set_compiler_environment_variables(pkg): if compiler.fc: os.environ['SPACK_FC'] = compiler.fc - os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) def set_build_environment_variables(pkg): - """This ensures a clean install environment when we build packages. + """ + This ensures a clean install environment when we build packages """ # Add spack build environment path with compiler wrappers first in # the path. We add both spack.env_path, which includes default # wrappers (cc, c++, f77, f90), AND a subdirectory containing # compiler-specific symlinks. The latter ensures that builds that # are sensitive to the *name* of the compiler see the right name - # when we're building wtih the wrappers. + # when we're building with the wrappers. # # Conflicts on case-insensitive systems (like "CC" and "cc") are # handled by putting one in the /case-insensitive @@ -296,23 +296,23 @@ def child_fun(): # do stuff build_env.fork(pkg, child_fun) - Forked processes are run with the build environemnt set up by + Forked processes are run with the build environment set up by spack.build_environment. This allows package authors to have - full control over the environment, etc. without offecting + full control over the environment, etc. without affecting other builds that might be executed in the same spack call. - If something goes wrong, the child process is expected toprint + If something goes wrong, the child process is expected to print the error and the parent process will exit with error as well. If things go well, the child exits and the parent carries on. """ try: pid = os.fork() - except OSError, e: + except OSError as e: raise InstallError("Unable to fork build process: %s" % e) if pid == 0: - # Give the child process the package's build environemnt. + # Give the child process the package's build environment. setup_package(pkg) try: @@ -323,7 +323,7 @@ def child_fun(): # which interferes with unit tests. os._exit(0) - except spack.error.SpackError, e: + except spack.error.SpackError as e: e.die() except: @@ -338,8 +338,7 @@ def child_fun(): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code." - .format(str(returncode))) + raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) class InstallError(spack.error.SpackError): From f5e8857c5e44719778e531bcc149c9fe228240b3 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Fri, 11 Mar 2016 09:51:12 -0700 Subject: [PATCH 252/339] + Rename variant 'sphinxbuild' to 'doc' as recommended in the discussion of PR#526. Also, remove a debug print statement that was accidentally committed. --- var/spack/repos/builtin/packages/cmake/package.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 806c37a68cc..cc93c7067c3 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -39,12 +39,12 @@ class Cmake(Package): variant('ncurses', default=True, description='Enables the build of the ncurses gui') variant('qt', default=False, description='Enables the build of cmake-gui') - variant('sphinxbuild', default=False, description='Enables the generation of html and man page documentation') + variant('doc', default=False, description='Enables the generation of html and man page documentation') depends_on('ncurses', when='+ncurses') depends_on('qt', when='+qt') - depends_on('python@2.7.11:', when='+sphinxbuild') - depends_on('py-sphinx', when='+sphinxbuild') + depends_on('python@2.7.11:', when='+doc') + depends_on('py-sphinx', when='+doc') def url_for_version(self, version): """Handle CMake's version-based custom URLs.""" @@ -58,8 +58,6 @@ def validate(self, spec): :raises RuntimeError: in case of inconsistencies """ - print spec - if '+qt' in spec and spec.satisfies('^qt@5.4.0'): msg = 'qt-5.4.0 has broken CMake modules.' raise RuntimeError(msg) @@ -75,7 +73,7 @@ def install(self, spec, prefix): if '+qt' in spec: options.append('--qt-gui') - if '+sphinxbuild' in spec: + if '+doc' in spec: options.append('--sphinx-html') options.append('--sphinx-man') From 1c7f754e5b8cd7740f3c4a91ee22a0354b40844a Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Fri, 11 Mar 2016 10:00:00 -0800 Subject: [PATCH 253/339] Invert and rename the `nobuild` option in package.yaml configs to `buildable`. --- lib/spack/docs/site_configuration.rst | 15 +++++++-------- lib/spack/spack/concretize.py | 8 ++++---- lib/spack/spack/config.py | 16 ++++++++-------- lib/spack/spack/test/mock_packages_test.py | 4 ++-- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 7ae541d4a39..3abfa21a9da 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -78,8 +78,7 @@ This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. If Spack is asked to build a package that uses one of these MPIs as a dependency, it will use the the pre-installed OpenMPI in -the given directory. This example also specifies that Spack should never -build its own OpenMPI via the ``nobuild: True`` option. +the given directory. Each ``packages.yaml`` begins with a ``packages:`` token, followed by a list of package names. To specify externals, add a ``paths`` @@ -111,16 +110,16 @@ be: paths: openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel - nobuild: True + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + buildable: False -The addition of the ``nobuild`` flag tells Spack that it should never build +The addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built -OpenMPI. Similar to ``paths``, ``nobuild`` is specified as a property under +OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under a package name. -The ``nobuild`` does not need to be paired with external packages. -It could also be used alone to forbid packages that may be +The ``buildable`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be buggy or otherwise undesirable. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2268084e568..8d29a03f932 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -69,12 +69,12 @@ def _valid_virtuals_and_externals(self, spec): packages = [spec] # For each candidate package, if it has externals add those to the candidates - # if it's a nobuild, then only add the externals. + # if it's not buildable, then only add the externals. candidates = [] all_compilers = spack.compilers.all_compilers() for pkg in packages: externals = spec_externals(pkg) - buildable = not is_spec_nobuild(pkg) + buildable = is_spec_buildable(pkg) if buildable: candidates.append((pkg, None)) for ext in externals: @@ -369,8 +369,8 @@ def __init__(self, spec): class NoBuildError(spack.error.SpackError): - """Raised when a package is configured with the nobuild option, but + """Raised when a package is configured with the buildable option False, but no satisfactory external versions can be found""" def __init__(self, spec): super(NoBuildError, self).__init__( - "The spec '%s' is configured as nobuild, and no matching external installs were found" % spec.name) + "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3a785fe6924..a21dd6dbe1e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -220,9 +220,9 @@ 'type' : 'array', 'default' : [], 'items' : { 'type' : 'string' } }, #compiler specs - 'nobuild': { + 'buildable': { 'type': 'boolean', - 'default': False, + 'default': True, }, 'providers': { 'type': 'object', @@ -557,15 +557,15 @@ def spec_externals(spec): return spec_locations -def is_spec_nobuild(spec): - """Return true if the spec pkgspec is configured as nobuild""" +def is_spec_buildable(spec): + """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') name = spec.name if not spec.name in allpkgs: - return False - if not 'nobuild' in allpkgs[spec.name]: - return False - return allpkgs[spec.name]['nobuild'] + return True + if not 'buildable' in allpkgs[spec.name]: + return True + return allpkgs[spec.name]['buildable'] class ConfigError(SpackError): pass diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 079cbcc1365..6d24a84150b 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -52,11 +52,11 @@ mock_packages_config = """\ packages: externaltool: - nobuild: True + buildable: False paths: externaltool@1.0%gcc@4.5.0: /path/to/external_tool externalvirtual: - nobuild: True + buildable: False paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc From bae03404f44ac60439e85e31a85bb6848db2a5e4 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 12:51:45 -0600 Subject: [PATCH 254/339] Documentation typo fixes --- lib/spack/docs/index.rst | 2 +- lib/spack/docs/packaging_guide.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 79757208c9a..d6ce52b747c 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -18,7 +18,7 @@ configurations can coexist on the same system. Most importantly, Spack is *simple*. It offers a simple *spec* syntax so that users can specify versions and configuration options concisely. Spack is also simple for package authors: package files -are writtin in pure Python, and specs allow package authors to +are written in pure Python, and specs allow package authors to maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index ef9fd89b62f..169899212db 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -419,7 +419,7 @@ directory to the directory containing the downloaded archive before it calls your ``install`` method. Within ``install``, the path to the downloaded archive is available as ``self.stage.archive_file``. -Here is an example snippet for packages distribuetd as self-extracting +Here is an example snippet for packages distributed as self-extracting archives. The example sets permissions on the downloaded file to make it executable, then runs it with some arguments. @@ -1556,12 +1556,12 @@ you ask for a particular spec. ``Concretization Policies`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A user may have certain perferrences for how packages should +A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -Spack can be configurated to prefer certain compilers, package +Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the @@ -1588,16 +1588,16 @@ At a high level, this example is specifying how packages should be concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by perfering gcc 4.9). +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each packages.yaml file begin with the string ``packages:`` and +Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` applies settings to each package. Underneath each package name is one or more components: ``compiler``, ``variants``, ``version``, or ``providers``. Each component has an ordered list of spec -``constraints``, with earlier entries in the list being prefered over +``constraints``, with earlier entries in the list being preferred over later entries. Sometimes a package installation may have constraints that forbid From 6ec65cd4ca1e6c2d346c949e21902680f27364c5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 15:03:37 -0600 Subject: [PATCH 255/339] Add GNU Octave package --- .../repos/builtin/packages/octave/package.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 var/spack/repos/builtin/packages/octave/package.py diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py new file mode 100644 index 00000000000..6810da6d98c --- /dev/null +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -0,0 +1,182 @@ +from spack import * + +class Octave(Package): + """GNU Octave is a high-level language, primarily intended for numerical + computations. It provides a convenient command line interface for solving + linear and nonlinear problems numerically, and for performing other + numerical experiments using a language that is mostly compatible with + Matlab. It may also be used as a batch-oriented language.""" + + homepage = "https://www.gnu.org/software/octave/" + url = "ftp://ftp.gnu.org/gnu/octave/octave-4.0.0.tar.gz" + + version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + + variant('readline', default=True) + variant('arpack', default=False) + variant('curl', default=False) + variant('fftw', default=False) + variant('fltk', default=False) + variant('fontconfig', default=False) + variant('freetype', default=False) + variant('glpk', default=False) + variant('gl2ps', default=False) + variant('gnuplot', default=False) + variant('magick', default=False) + variant('hdf5', default=False) + variant('jdk', default=False) + variant('llvm', default=False) + variant('opengl', default=False) + variant('qhull', default=False) + variant('qrupdate', default=False) + variant('qscintilla', default=False) + variant('qt', default=False) + variant('suiteparse', default=False) + variant('zlib', default=False) + + # Required dependencies + depends_on('blas') + depends_on('lapack') + depends_on('pcre') + + # Strongly recommended dependencies + depends_on('readline', when='+readline') + + # Optional dependencies + depends_on('arpack', when='+arpack') + depends_on('curl', when='+curl') + depends_on('fftw@3', when='+fftw') + depends_on('fltk', when='+fltk') + depends_on('fontconfig', when='+fontconfig') + depends_on('freetype', when='+freetype') + depends_on('glpk', when='+glpk') + #depends_on('gl2ps', when='+gl2ps') + depends_on('gnuplot', when='+gnuplot') + depends_on('ImageMagick', when='+magick') + depends_on('hdf5', when='+hdf5') + depends_on('jdk', when='+jdk') + depends_on('llvm', when='+llvm') + #depends_on('opengl', when='+opengl') + depends_on('qhull', when='+qhull') + #depends_on('qrupdate', when='+qrupdate') + #depends_on('qscintilla', when='+qscintilla) + depends_on('qt', when='+qt') + depends_on('SuiteSparse', when='suitesparse') + depends_on('zlib', when='+zlib') + + + def install(self, spec, prefix): + config_args = [ + "--prefix=%s" % prefix + ] + + # Required dependencies + config_args.extend([ + "--with-blas=%s" % spec['blas'].prefix.lib, + "--with-lapack=%s" % spec['lapack'].prefix.lib + ]) + + # Strongly recommended dependencies + if '+readline' in spec: + config_args.append('--enable-readline') + else: + config_args.append('--disable-readline') + + # Optional dependencies + if '+arpack' in spec: + config_args.extend([ + "--with-arpack-includedir=%s" % spec['arpack'].prefix.include, + "--with-arpack-libdir=%s" % spec['arpack'].prefix.lib + ]) + else: + config_args.append("--without-arpack") + + if '+curl' in spec: + config_args.extend([ + "--with-curl-includedir=%s" % spec['curl'].prefix.include, + "--with-curl-libdir=%s" % spec['curl'].prefix.lib + ]) + else: + config_args.append("--without-curl") + + if '+fftw' in spec: + config_args.extend([ + "--with-fftw3-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3-libdir=%s" % spec['fftw'].prefix.lib, + "--with-fftw3f-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3f-libdir=%s" % spec['fftw'].prefix.lib + ]) + else: + config_args.extend([ + "--without-fftw3", + "--without-fftw3f" + ]) + + if '+fltk' in spec: + config_args.extend([ + "--with-fltk-prefix=%s" % spec['fltk'].prefix, + "--with-fltk-exec-prefix=%s" % spec['fltk'].prefix + ]) + else: + config_args.append("--without-fltk") + + if '+glpk' in spec: + config_args.extend([ + "--with-glpk-includedir=%s" % spec['glpk'].prefix.include + "--with-glpk-libdir=%s" % spec['glpk'].prefix.lib + ]) + else: + config_args.append("--without-glpk") + + if '+magick' in spec: + config_args.append("--with-magick=%s" % spec['ImageMagick'].prefix.lib) + + if '+hdf5' in spec: + config_args.extend([ + "--with-hdf5-includedir=%s" % spec['hdf5'].prefix.include, + "--with-hdf5-libdir=%s" % spec['hdf5'].prefix.lib + ]) + else: + config_args.append("--without-hdf5") + + if '+jdk' in spec: + config_args.extend([ + "--with-java-homedir=%s" % spec['jdk'].prefix, + "--with-java-includedir=%s" % spec['jdk'].prefix.include, + "--with-java-libdir=%s" % spec['jdk'].prefix.lib + ]) + + #if '~opengl' in spec: + # config_args.extend([ + # "--without-opengl", + # "--without-framework-opengl" + # ]) + + if '+qhull' in spec: + config_args.extend([ + "--with-qhull-includedir=%s" % spec['qhull'].prefix.include, + "--with-qhull-libdir=%s" % spec['qhull'].prefix.lib + ]) + else: + config_args.append("--without-qhull") + + #if '+qrupdate' in spec: + # config_args.extend([ + # "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + # "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib + # ]) + #else: + # config_args.append("--without-qrupdate") + + if '+zlib' in spec: + config_args.extend([ + "--with-z-includedir=%s" % spec['zlib'].prefix.include, + "--with-z-libdir=%s" % spec['zlib'].prefix.lib + ]) + else: + config_args.append("--without-z") + + configure(*config_args) + + make() + make("install") From 1179217334386984211645d508d8e02692297183 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Mar 2016 13:33:56 -0800 Subject: [PATCH 256/339] Add compiler info and a simple libdwarf build to the checks. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ab379be4867..1bed6b08740 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,10 @@ before_install: script: - . share/spack/setup-env.sh + - spack compilers + - spack config get compilers - spack test + - spack install -v libdwarf notifications: email: From 145390c7f3a9de6e679f56dc9b275973a459de91 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 11 Mar 2016 16:57:37 -0600 Subject: [PATCH 257/339] Add gl2ps and qrupdate packages --- .../repos/builtin/packages/gl2ps/package.py | 18 ++++++++++++++ .../repos/builtin/packages/octave/package.py | 24 +++++++++---------- .../builtin/packages/qrupdate/package.py | 18 ++++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 var/spack/repos/builtin/packages/gl2ps/package.py create mode 100644 var/spack/repos/builtin/packages/qrupdate/package.py diff --git a/var/spack/repos/builtin/packages/gl2ps/package.py b/var/spack/repos/builtin/packages/gl2ps/package.py new file mode 100644 index 00000000000..cb376b3f03b --- /dev/null +++ b/var/spack/repos/builtin/packages/gl2ps/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Gl2ps(Package): + """GL2PS is a C library providing high quality vector output for any + OpenGL application.""" + + homepage = "http://www.geuz.org/gl2ps/" + url = "http://geuz.org/gl2ps/src/gl2ps-1.3.9.tgz" + + version('1.3.9', '377b2bcad62d528e7096e76358f41140') + + depends_on("libpng") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 6810da6d98c..99847e1dbee 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -50,7 +50,7 @@ class Octave(Package): depends_on('fontconfig', when='+fontconfig') depends_on('freetype', when='+freetype') depends_on('glpk', when='+glpk') - #depends_on('gl2ps', when='+gl2ps') + depends_on('gl2ps', when='+gl2ps') depends_on('gnuplot', when='+gnuplot') depends_on('ImageMagick', when='+magick') depends_on('hdf5', when='+hdf5') @@ -58,10 +58,10 @@ class Octave(Package): depends_on('llvm', when='+llvm') #depends_on('opengl', when='+opengl') depends_on('qhull', when='+qhull') - #depends_on('qrupdate', when='+qrupdate') + depends_on('qrupdate', when='+qrupdate') #depends_on('qscintilla', when='+qscintilla) depends_on('qt', when='+qt') - depends_on('SuiteSparse', when='suitesparse') + depends_on('SuiteSparse', when='+suitesparse') depends_on('zlib', when='+zlib') @@ -72,7 +72,7 @@ def install(self, spec, prefix): # Required dependencies config_args.extend([ - "--with-blas=%s" % spec['blas'].prefix.lib, + "--with-blas=%s" % spec['blas'].prefix.lib, "--with-lapack=%s" % spec['lapack'].prefix.lib ]) @@ -122,7 +122,7 @@ def install(self, spec, prefix): if '+glpk' in spec: config_args.extend([ - "--with-glpk-includedir=%s" % spec['glpk'].prefix.include + "--with-glpk-includedir=%s" % spec['glpk'].prefix.include, "--with-glpk-libdir=%s" % spec['glpk'].prefix.lib ]) else: @@ -160,13 +160,13 @@ def install(self, spec, prefix): else: config_args.append("--without-qhull") - #if '+qrupdate' in spec: - # config_args.extend([ - # "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, - # "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib - # ]) - #else: - # config_args.append("--without-qrupdate") + if '+qrupdate' in spec: + config_args.extend([ + "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib + ]) + else: + config_args.append("--without-qrupdate") if '+zlib' in spec: config_args.extend([ diff --git a/var/spack/repos/builtin/packages/qrupdate/package.py b/var/spack/repos/builtin/packages/qrupdate/package.py new file mode 100644 index 00000000000..aff44bb2d80 --- /dev/null +++ b/var/spack/repos/builtin/packages/qrupdate/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Qrupdate(Package): + """qrupdate is a Fortran library for fast updates of QR and + Cholesky decompositions.""" + + homepage = "http://sourceforge.net/projects/qrupdate/" + url = "https://downloads.sourceforge.net/qrupdate/qrupdate-1.1.2.tar.gz" + + version('1.1.2', '6d073887c6e858c24aeda5b54c57a8c4') + + depends_on("openblas") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") From 90f2e40ff92077da1f1b9b4d9aa317b89aa07960 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:28:16 -0500 Subject: [PATCH 258/339] Added comment to Emacs. --- var/spack/repos/builtin/packages/emacs/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py index 09eb05d5a76..caa264857e0 100644 --- a/var/spack/repos/builtin/packages/emacs/package.py +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -12,6 +12,8 @@ class Emacs(Package): # GTK or other widget library # libtiff, png, etc. # For now, we assume the system provides all that stuff. + # For Ubuntu 14.04 LTS: + # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev def install(self, spec, prefix): configure('--prefix=%s' % prefix) From 7cd478418d5b8a204a8e6b9a5307ecf71ea83939 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Fri, 11 Mar 2016 23:28:36 -0500 Subject: [PATCH 259/339] New version of LAPACK --- var/spack/repos/builtin/packages/netlib-lapack/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index fb6b99e27c3..741f4af4215 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -12,6 +12,7 @@ class NetlibLapack(Package): homepage = "http://www.netlib.org/lapack/" url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + version('3.6.0', 'f2f6c67134e851fe189bb3ca1fbb5101') version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') version('3.4.1', '44c3869c38c8335c2b9c2a8bb276eb55') From 3383486adc86ba03456aae3703c0176620888e77 Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:38:05 -0400 Subject: [PATCH 260/339] Fixed typo bug. Made error comment more explicit --- lib/spack/spack/cmd/diy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 2c3a8761ab6..199362d915c 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -75,8 +75,8 @@ def diy(self, args): edit_package(spec.name, spack.repo.first_repo(), None, True) return - if not spec.version.concrete: - tty.die("spack diy spec must have a single, concrete version.") + if not spec.versions.concrete: + tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) From 5c865b9b702695797ab29263fb629b53e56983fb Mon Sep 17 00:00:00 2001 From: Elizabeth F Date: Sun, 13 Mar 2016 19:42:15 -0400 Subject: [PATCH 261/339] Fixed typo in typo fix. --- lib/spack/spack/cmd/diy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 199362d915c..45f13e44639 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -76,7 +76,7 @@ def diy(self, args): return if not spec.versions.concrete: - tty.die("spack spconfig spec must have a single, concrete version. Did you forget a package version number?") + tty.die("spack diy spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) From 8b715d9c3fabdf4658ff4c0c500d01f74e7ed64e Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sun, 13 Mar 2016 21:14:41 -0400 Subject: [PATCH 262/339] Update tmux --- var/spack/repos/builtin/packages/tmux/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py index 23d36db4275..f2067d13665 100644 --- a/var/spack/repos/builtin/packages/tmux/package.py +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -7,10 +7,11 @@ class Tmux(Package): do a lot more. """ - homepage = "http://tmux.sourceforge.net" - url = "http://downloads.sourceforge.net/project/tmux/tmux/tmux-1.9/tmux-1.9a.tar.gz" + homepage = "http://tmux.github.io" + url = "https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz" version('1.9a', 'b07601711f96f1d260b390513b509a2d') + version('2.1', '74a2855695bccb51b6e301383ad4818c') depends_on('libevent') depends_on('ncurses') From 003fd4d834d3e06ea89c7f3c2fa13241b0730f06 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 04:55:30 -0700 Subject: [PATCH 263/339] Optimize __eq__ and __ne__ in key_ordering - use `is` when possible before calling `_cmp_key()` --- lib/spack/llnl/util/lang.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 1c4d1ed623e..13d301f84e6 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -235,11 +235,11 @@ def setter(name, value): if not has_method(cls, '_cmp_key'): raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__) - setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key()) + setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) setter('__lt__', lambda s,o: o is not None and s._cmp_key() < o._cmp_key()) setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key()) - setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key()) + setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) setter('__gt__', lambda s,o: o is None or s._cmp_key() > o._cmp_key()) setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key()) From 05c761dee9c86faf9ce6d5b98ae57c8737694898 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 04:59:29 -0700 Subject: [PATCH 264/339] Add `package_class` method to spec. - Shouldn't call .package from within things like normalize() and concretize() beacuse spec may be inconsistent. - Add `.package_class` property so that we can get at package metadata without constructing a Package with a Spec. - should be faster than `.package` was, anyway. Use where possible. --- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/repository.py | 9 +++++++-- lib/spack/spack/spec.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8d29a03f932..445ecd8896d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -238,7 +238,7 @@ def concretize_variants(self, spec): the default variants from the package specification. """ changed = False - for name, variant in spec.package.variants.items(): + for name, variant in spec.package_class.variants.items(): if name not in spec.variants: spec.variants[name] = spack.spec.VariantSpec(name, variant.default) changed = True diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 3c3ba08bcc2..d2fdc937f7a 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -316,6 +316,11 @@ def get(self, spec, new=False): return self.repo_for_pkg(spec).get(spec) + def get_pkg_class(self, pkg_name): + """Find a class for the spec's package and return the class object.""" + return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) + + @_autospec def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. @@ -550,7 +555,7 @@ def get(self, spec, new=False): key = hash(spec) if new or key not in self._instances: - package_class = self._get_pkg_class(spec.name) + package_class = self.get_pkg_class(spec.name) try: copy = spec.copy() # defensive copy. Package owns its spec. self._instances[key] = package_class(copy) @@ -715,7 +720,7 @@ def _get_pkg_module(self, pkg_name): return self._modules[pkg_name] - def _get_pkg_class(self, pkg_name): + def get_pkg_class(self, pkg_name): """Get the class for the package out of its module. First loads (or fetches from cache) a module for the diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c045e803655..573e288d174 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -353,7 +353,7 @@ def constrain(self, other): @property def concrete(self): return self.spec._concrete or all( - v in self for v in self.spec.package.variants) + v in self for v in self.spec.package_class.variants) def copy(self): @@ -498,6 +498,14 @@ def package(self): return spack.repo.get(self) + @property + def package_class(self): + """Internal package call gets only the class object for a package. + Use this to just get package metadata. + """ + return spack.repo.get_pkg_class(self.name) + + @property def virtual(self): """Right now, a spec is virtual if no package exists with its name. @@ -1161,7 +1169,7 @@ def validate_names(self): # Ensure that variants all exist. for vname, variant in spec.variants.items(): - if vname not in spec.package.variants: + if vname not in spec.package_class.variants: raise UnknownVariantError(spec.name, vname) From f45b8b1083e5f628dd31fa4b7b873b6df7119d0e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 05:02:50 -0700 Subject: [PATCH 265/339] Add some tests for packages with multiple virtual dependencies. - Added mock `hypre` package, depends on `lapack` and `blas`. - test cases where some packages provide both `lapack` and `blas`, but others do not. --- lib/spack/spack/test/concretize.py | 29 +++++++++++++- .../builtin.mock/packages/hypre/package.py | 39 +++++++++++++++++++ .../packages/openblas-with-lapack/package.py | 38 ++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 var/spack/repos/builtin.mock/packages/hypre/package.py create mode 100644 var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 07828d8ea6c..f264faf17a2 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -142,6 +142,34 @@ def test_concretize_with_provides_when(self): for spec in spack.repo.providers_for('mpi@3'))) + def test_concretize_two_virtuals(self): + """Test a package with multiple virtual dependencies.""" + s = Spec('hypre').concretize() + + + def test_concretize_two_virtuals_with_one_bound(self): + """Test a package with multiple virtual dependencies and one preset.""" + s = Spec('hypre ^openblas').concretize() + + + def test_concretize_two_virtuals_with_two_bound(self): + """Test a package with multiple virtual dependencies and two of them preset.""" + s = Spec('hypre ^openblas ^netlib-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both.""" + s = Spec('hypre ^openblas-with-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both, and another conflicting package that provides one.""" + s = Spec('hypre ^openblas-with-lapack ^netlib-lapack') + self.assertRaises(spack.spec.MultipleProviderError, s.concretize) + + def test_virtual_is_fully_expanded_for_callpath(self): # force dependence on fake "zmpi" by asking for MPI 10.0 spec = Spec('callpath ^mpi@10.0') @@ -281,4 +309,3 @@ def test_find_spec_none(self): Spec('d')), Spec('e')) self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) - diff --git a/var/spack/repos/builtin.mock/packages/hypre/package.py b/var/spack/repos/builtin.mock/packages/hypre/package.py new file mode 100644 index 00000000000..f69f16d2cc3 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/hypre/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Hypre(Package): + """Hypre is included here as an example of a package that depends on + both LAPACK and BLAS.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + depends_on('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py new file mode 100644 index 00000000000..509bfb71e56 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class OpenblasWithLapack(Package): + """Dummy version of OpenBLAS that also provides LAPACK, for testing.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('lapack') + provides('blas') + + def install(self, spec, prefix): + pass From f2761270f3c0506a689b484b0e12d7d6e9f4300d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Mar 2016 05:04:01 -0700 Subject: [PATCH 266/339] Make concretization less greedy: add backtracking for virtuals. - `_expand_virtual_packages` now gets a candidate list and will try all the candidates. - Good news: If the first virtual in the list conflicts with something else in the spec, we'll keep trying until we find a good one. - Bad news: Only looks as far as the next normalize(); can't see conflicts further ahead than that if they're inevitable some other virtual expansion. - Refactor `concretize.py` to keep all the nasty spec graph stitching in `spec.py`. This is more similar to before externals support. - `concretize.py` now just returns a list of candidates sorted by ABI compatibility to `_expand_virtual_packages`, and `spec.py` handles testing the candidates. - Refactor the way external paths are handled in `config.py` and `concretize.py`: - previously, `spec_externals` returned spec/path pairs. Now it returns specs with `external` set. Makes code in `concretize.py` more natural. --- lib/spack/spack/concretize.py | 130 ++++++++++++++-------------------- lib/spack/spack/config.py | 17 +++-- lib/spack/spack/spec.py | 129 +++++++++++++++++++++++++++------ 3 files changed, 169 insertions(+), 107 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 445ecd8896d..8083f919828 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -51,10 +51,10 @@ class DefaultConcretizer(object): """ def _valid_virtuals_and_externals(self, spec): - """Returns a list of spec/external-path pairs for both virtuals and externals - that can concretize this spec.""" - # Get a list of candidate packages that could satisfy this spec - packages = [] + """Returns a list of candidate virtual dep providers and external + packages that coiuld be used to concretize a spec.""" + # First construct a list of concrete candidates to replace spec with. + candidates = [spec] if spec.virtual: providers = spack.repo.providers_for(spec) if not providers: @@ -64,96 +64,72 @@ def _valid_virtuals_and_externals(self, spec): if not spec_w_preferred_providers: spec_w_preferred_providers = spec provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) - packages = sorted(providers, cmp=provider_cmp) - else: - packages = [spec] + candidates = sorted(providers, cmp=provider_cmp) - # For each candidate package, if it has externals add those to the candidates - # if it's not buildable, then only add the externals. - candidates = [] - all_compilers = spack.compilers.all_compilers() - for pkg in packages: - externals = spec_externals(pkg) - buildable = is_spec_buildable(pkg) - if buildable: - candidates.append((pkg, None)) + # For each candidate package, if it has externals, add those to the usable list. + # if it's not buildable, then *only* add the externals. + usable = [] + for cspec in candidates: + if is_spec_buildable(cspec): + usable.append(cspec) + externals = spec_externals(cspec) for ext in externals: - if ext[0].satisfies(spec): - candidates.append(ext) - if not candidates: + if ext.satisfies(spec): + usable.append(ext) + + # If nothing is in the usable list now, it's because we aren't + # allowed to build anything. + if not usable: raise NoBuildError(spec) def cmp_externals(a, b): - if a[0].name != b[0].name: - #We're choosing between different providers. Maintain order from above sort + if a.name != b.name: + # We're choosing between different providers, so + # maintain order from provider sort return candidates.index(a) - candidates.index(b) - result = cmp_specs(a[0], b[0]) + + result = cmp_specs(a, b) if result != 0: return result - if not a[1] and b[1]: - return 1 - if not b[1] and a[1]: - return -1 - return cmp(a[1], b[1]) - candidates = sorted(candidates, cmp=cmp_externals) - return candidates + # prefer external packages to internal packages. + if a.external is None or b.external is None: + return -cmp(a.external, b.external) + else: + return cmp(a.external, b.external) + + usable.sort(cmp=cmp_externals) + return usable - def concretize_virtual_and_external(self, spec): - """From a list of candidate virtual and external packages, concretize to one that - is ABI compatible with the rest of the DAG.""" + def choose_virtual_or_external(self, spec): + """Given a list of candidate virtual and external packages, try to + find one that is most ABI compatible. + """ candidates = self._valid_virtuals_and_externals(spec) if not candidates: - return False + return candidates - # Find the nearest spec in the dag that has a compiler. We'll use that - # spec to test compiler compatibility. - other_spec = find_spec(spec, lambda(x): x.compiler) - if not other_spec: - other_spec = spec.root + # Find the nearest spec in the dag that has a compiler. We'll + # use that spec to calibrate compiler compatibility. + abi_exemplar = find_spec(spec, lambda(x): x.compiler) + if not abi_exemplar: + abi_exemplar = spec.root - # Choose an ABI-compatible candidate, or the first match otherwise. - candidate = None - if other_spec: - candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec)), None) - if not candidate: - # Try a looser ABI matching - candidate = next((c for c in candidates if spack.abi.compatible(c[0], other_spec, loose=True)), None) - if not candidate: - # No ABI matches. Pick the top choice based on the orignal preferences. - candidate = candidates[0] - candidate_spec = candidate[0] - external = candidate[1] - changed = False + # Make a list including ABI compatibility of specs with the exemplar. + strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates] + loose = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates] + keys = zip(strict, loose, candidates) - # If we're external then trim the dependencies - if external: - if (spec.dependencies): - changed = True - spec.dependencies = DependencyMap() - candidate_spec.dependencies = DependencyMap() + # Sort candidates from most to least compatibility. + # Note: + # 1. We reverse because True > False. + # 2. Sort is stable, so c's keep their order. + keys.sort(key=lambda k:k[:2], reverse=True) - def fequal(candidate_field, spec_field): - return (not candidate_field) or (candidate_field == spec_field) - if (fequal(candidate_spec.name, spec.name) and - fequal(candidate_spec.versions, spec.versions) and - fequal(candidate_spec.compiler, spec.compiler) and - fequal(candidate_spec.architecture, spec.architecture) and - fequal(candidate_spec.dependencies, spec.dependencies) and - fequal(candidate_spec.variants, spec.variants) and - fequal(external, spec.external)): - return changed - - # Refine this spec to the candidate. - if spec.virtual: - spec._replace_with(candidate_spec) - changed = True - if spec._dup(candidate_spec, deps=False, cleardeps=False): - changed = True - spec.external = external - - return changed + # Pull the candidates back out and return them in order + candidates = [c for s,l,c in keys] + return candidates def concretize_version(self, spec): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index a21dd6dbe1e..6afd69b3ac7 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -539,22 +539,25 @@ def print_section(section): def spec_externals(spec): - """Return a list of spec, directory pairs for each external location for spec""" + """Return a list of external specs (with external directory path filled in), + one for each known external installation.""" allpkgs = get_config('packages') name = spec.name - spec_locations = [] + external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) if not pkg_paths: return [] - for pkg,path in pkg_paths.iteritems(): - if not spec.satisfies(pkg): - continue + for external_spec, path in pkg_paths.iteritems(): if not path: + # skip entries without paths (avoid creating extra Specs) continue - spec_locations.append( (spack.spec.Spec(pkg), path) ) - return spec_locations + + external_spec = spack.spec.Spec(external_spec, external=path) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs def is_spec_buildable(spec): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 573e288d174..d04135860e7 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -418,9 +418,11 @@ def __init__(self, spec_like, *dep_like, **kwargs): # cases we've read them from a file want to assume normal. # This allows us to manipulate specs that Spack doesn't have # package.py files for. - self._normal = kwargs.get('normal', False) + self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) - self.external = None + + # Allow a spec to be constructed with an external path. + self.external = kwargs.get('external', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -794,8 +796,30 @@ def _replace_with(self, concrete): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): + # remove self from all dependents. del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + + # add the replacement, unless it is already a dep of dependent. + if concrete.name not in dependent.dependencies: + dependent._add_dependency(concrete) + + + def _replace_node(self, replacement): + """Replace this spec with another. + + Connects all dependents of this spec to its replacement, and + disconnects this spec from any dependencies it has. New spec + will have any dependencies the replacement had, and may need + to be normalized. + + """ + for name, dependent in self.dependents.items(): + del dependent.dependencies[self.name] + dependent._add_dependency(replacement) + + for name, dep in self.dependencies.items(): + del dep.dependents[self.name] + del self.dependencies[dep.name] def _expand_virtual_packages(self): @@ -815,18 +839,80 @@ def _expand_virtual_packages(self): this are infrequent, but should implement this before it is a problem. """ + # Make an index of stuff this spec already provides + self_index = ProviderIndex(self.traverse(), restrict=True) + changed = False done = False while not done: done = True for spec in list(self.traverse()): - if spack.concretizer.concretize_virtual_and_external(spec): - done = False + replacement = None + if spec.virtual: + replacement = self._find_provider(spec, self_index) + if replacement: + # TODO: may break if in-place on self but + # shouldn't happen if root is traversed first. + spec._replace_with(replacement) + done=False + break + + if not replacement: + # Get a list of possible replacements in order of preference. + candidates = spack.concretizer.choose_virtual_or_external(spec) + + # Try the replacements in order, skipping any that cause + # satisfiability problems. + for replacement in candidates: + if replacement is spec: + break + + # Replace spec with the candidate and normalize + copy = self.copy() + copy[spec.name]._dup(replacement.copy(deps=False)) + + try: + # If there are duplicate providers or duplicate provider + # deps, consolidate them and merge constraints. + copy.normalize(force=True) + break + except SpecError as e: + # On error, we'll try the next replacement. + continue + + # If replacement is external then trim the dependencies + if replacement.external: + if (spec.dependencies): + changed = True + spec.dependencies = DependencyMap() + replacement.dependencies = DependencyMap() + + # TODO: could this and the stuff in _dup be cleaned up? + def feq(cfield, sfield): + return (not cfield) or (cfield == sfield) + + if replacement is spec or (feq(replacement.name, spec.name) and + feq(replacement.versions, spec.versions) and + feq(replacement.compiler, spec.compiler) and + feq(replacement.architecture, spec.architecture) and + feq(replacement.dependencies, spec.dependencies) and + feq(replacement.variants, spec.variants) and + feq(replacement.external, spec.external)): + continue + + # Refine this spec to the candidate. This uses + # replace_with AND dup so that it can work in + # place. TODO: make this more efficient. + if spec.virtual: + spec._replace_with(replacement) + changed = True + if spec._dup(replacement, deps=False, cleardeps=False): changed = True - # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merge constraints. - changed |= self.normalize(force=True) + self_index.update(spec) + done=False + break + return changed @@ -850,7 +936,7 @@ def concretize(self): force = False while changed: - changes = (self.normalize(force=force), + changes = (self.normalize(force), self._expand_virtual_packages(), self._concretize_helper()) changed = any(changes) @@ -976,8 +1062,8 @@ def _evaluate_dependency_conditions(self, name): def _find_provider(self, vdep, provider_index): """Find provider for a virtual spec in the provider index. - Raise an exception if there is a conflicting virtual - dependency already in this spec. + Raise an exception if there is a conflicting virtual + dependency already in this spec. """ assert(vdep.virtual) providers = provider_index.providers_for(vdep) @@ -1018,17 +1104,14 @@ def _merge_dependency(self, dep, visited, spec_deps, provider_index): """ changed = False - # If it's a virtual dependency, try to find a provider and - # merge that. + # If it's a virtual dependency, try to find an existing + # provider in the spec, and merge that. if dep.virtual: visited.add(dep.name) provider = self._find_provider(dep, provider_index) if provider: dep = provider - else: - # if it's a real dependency, check whether it provides - # something already required in the spec. index = ProviderIndex([dep], restrict=True) for vspec in (v for v in spec_deps.values() if v.virtual): if index.providers_for(vspec): @@ -1125,13 +1208,14 @@ def normalize(self, force=False): # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) - # Initialize index of virtual dependency providers - index = ProviderIndex(spec_deps.values(), restrict=True) + # Initialize index of virtual dependency providers if + # concretize didn't pass us one already + provider_index = ProviderIndex(spec_deps.values(), restrict=True) # traverse the package DAG and fill out dependencies according # to package files & their 'when' specs visited = set() - any_change = self._normalize_helper(visited, spec_deps, index) + any_change = self._normalize_helper(visited, spec_deps, provider_index) # If there are deps specified but not visited, they're not # actually deps of this package. Raise an error. @@ -1410,13 +1494,12 @@ def _dup(self, other, **kwargs): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ - # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): - changed = (self.name != other.name and self.versions != other.versions and \ - self.architecture != other.architecture and self.compiler != other.compiler and \ - self.variants != other.variants and self._normal != other._normal and \ + changed = (self.name != other.name and self.versions != other.versions and + self.architecture != other.architecture and self.compiler != other.compiler and + self.variants != other.variants and self._normal != other._normal and self.concrete != other.concrete and self.external != other.external) # Local node attributes get copied first. From f9923452b3365e1472a4f1bd63712c57fea0ee0d Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 14 Mar 2016 14:35:48 +0100 Subject: [PATCH 267/339] environment : added machinery to collect modifications to the environment and apply them later --- lib/spack/spack/environment.py | 157 ++++++++++++++++++++++++++++ lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/environment.py | 50 +++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/environment.py create mode 100644 lib/spack/spack/test/environment.py diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py new file mode 100644 index 00000000000..7d3d7af0de1 --- /dev/null +++ b/lib/spack/spack/environment.py @@ -0,0 +1,157 @@ +import os +import os.path +import collections + + +class SetEnv(object): + def __init__(self, name, value, **kwargs): + self.name = name + self.value = value + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + os.environ[self.name] = str(self.value) + + +class UnsetEnv(object): + def __init__(self, name, **kwargs): + self.name = name + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + + +class AppendPath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + # TODO : Check if this is a valid directory name + directories.append(os.path.normpath(self.path)) + os.environ[self.name] = ':'.join(directories) + + +class PrependPath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + # TODO : Check if this is a valid directory name + directories = [os.path.normpath(self.path)] + directories + os.environ[self.name] = ':'.join(directories) + + +class RemovePath(object): + def __init__(self, name, path, **kwargs): + self.name = name + self.path = path + for key, value in kwargs.items(): + setattr(self, key, value) + + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.path)] + os.environ[self.name] = ':'.join(directories) + + +class EnvironmentModifications(object): + """ + Keeps track of requests to modify the current environment + """ + + def __init__(self): + self.env_modifications = [] + + def __iter__(self): + return iter(self.env_modifications) + + def set_env(self, name, value, **kwargs): + """ + Stores in the current object a request to set an environment variable + + Args: + name: name of the environment variable to be set + value: value of the environment variable + """ + item = SetEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def unset_env(self, name, **kwargs): + """ + Stores in the current object a request to unset an environment variable + + Args: + name: name of the environment variable to be set + """ + item = UnsetEnv(name, **kwargs) + self.env_modifications.append(item) + + def append_path(self, name, path, **kwargs): + """ + Stores in the current object a request to append a path to a path list + + Args: + name: name of the path list in the environment + path: path to be appended + """ + item = AppendPath(name, path, **kwargs) + self.env_modifications.append(item) + + def prepend_path(self, name, path, **kwargs): + """ + Same as `append_path`, but the path is pre-pended + + Args: + name: name of the path list in the environment + path: path to be pre-pended + """ + item = PrependPath(name, path, **kwargs) + self.env_modifications.append(item) + + def remove_path(self, name, path, **kwargs): + """ + Stores in the current object a request to remove a path from a path list + + Args: + name: name of the path list in the environment + path: path to be removed + """ + item = RemovePath(name, path, **kwargs) + self.env_modifications.append(item) + + +def validate_environment_modifications(env): + modifications = collections.defaultdict(list) + for item in env: + modifications[item.name].append(item) + return modifications + + +def apply_environment_modifications(env): + """ + Modifies the current environment according to the request in env + + Args: + env: object storing modifications to the environment + """ + modifications = validate_environment_modifications(env) + + # Cycle over the environment variables that will be modified + for variable, actions in modifications.items(): + # Execute all the actions in the order they were issued + for x in actions: + x.execute() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index d5d8b647650..cd842561e6c 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -66,7 +66,8 @@ 'database', 'namespace_trie', 'yaml', - 'sbang'] + 'sbang', + 'environment'] def list_tests(): diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py new file mode 100644 index 00000000000..dff3863d322 --- /dev/null +++ b/lib/spack/spack/test/environment.py @@ -0,0 +1,50 @@ +import unittest +import os +from spack.environment import EnvironmentModifications, apply_environment_modifications + + +class EnvironmentTest(unittest.TestCase): + def setUp(self): + os.environ.clear() + os.environ['UNSET_ME'] = 'foo' + os.environ['EMPTY_PATH_LIST'] = '' + os.environ['PATH_LIST'] = '/path/second:/path/third' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + + def test_set_env(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value') + env.set_env('B', 3) + apply_environment_modifications(env) + self.assertEqual('dummy value', os.environ['A']) + self.assertEqual(str(3), os.environ['B']) + + def test_unset_env(self): + env = EnvironmentModifications() + self.assertEqual('foo', os.environ['UNSET_ME']) + env.unset_env('UNSET_ME') + apply_environment_modifications(env) + self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') + + def test_path_manipulation(self): + env = EnvironmentModifications() + + env.append_path('PATH_LIST', '/path/last') + env.prepend_path('PATH_LIST', '/path/first') + + env.append_path('EMPTY_PATH_LIST', '/path/middle') + env.append_path('EMPTY_PATH_LIST', '/path/last') + env.prepend_path('EMPTY_PATH_LIST', '/path/first') + + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle') + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last') + env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first') + + env.remove_path('REMOVE_PATH_LIST', '/remove/this') + env.remove_path('REMOVE_PATH_LIST', '/duplicate/') + + apply_environment_modifications(env) + self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) From 0d9a6d3c25de2b8490622c49362b09f706459bc0 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 14 Mar 2016 14:19:30 -0500 Subject: [PATCH 268/339] Updates to qrupdate --- .../repos/builtin/packages/octave/package.py | 17 +++++++++-------- .../repos/builtin/packages/qrupdate/package.py | 10 +++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 99847e1dbee..38b355159dd 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -12,6 +12,7 @@ class Octave(Package): version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + # Variants variant('readline', default=True) variant('arpack', default=False) variant('curl', default=False) @@ -45,7 +46,7 @@ class Octave(Package): # Optional dependencies depends_on('arpack', when='+arpack') depends_on('curl', when='+curl') - depends_on('fftw@3', when='+fftw') + depends_on('fftw', when='+fftw') depends_on('fltk', when='+fltk') depends_on('fontconfig', when='+fontconfig') depends_on('freetype', when='+freetype') @@ -56,10 +57,10 @@ class Octave(Package): depends_on('hdf5', when='+hdf5') depends_on('jdk', when='+jdk') depends_on('llvm', when='+llvm') - #depends_on('opengl', when='+opengl') + #depends_on('opengl', when='+opengl') # TODO: add package depends_on('qhull', when='+qhull') depends_on('qrupdate', when='+qrupdate') - #depends_on('qscintilla', when='+qscintilla) + #depends_on('qscintilla', when='+qscintilla) # TODO: add package depends_on('qt', when='+qt') depends_on('SuiteSparse', when='+suitesparse') depends_on('zlib', when='+zlib') @@ -146,11 +147,11 @@ def install(self, spec, prefix): "--with-java-libdir=%s" % spec['jdk'].prefix.lib ]) - #if '~opengl' in spec: - # config_args.extend([ - # "--without-opengl", - # "--without-framework-opengl" - # ]) + if '~opengl' in spec: + config_args.extend([ + "--without-opengl", + "--without-framework-opengl" + ]) if '+qhull' in spec: config_args.extend([ diff --git a/var/spack/repos/builtin/packages/qrupdate/package.py b/var/spack/repos/builtin/packages/qrupdate/package.py index aff44bb2d80..5374d02c971 100644 --- a/var/spack/repos/builtin/packages/qrupdate/package.py +++ b/var/spack/repos/builtin/packages/qrupdate/package.py @@ -9,10 +9,10 @@ class Qrupdate(Package): version('1.1.2', '6d073887c6e858c24aeda5b54c57a8c4') - depends_on("openblas") + depends_on("blas") + depends_on("lapack") def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") + # Build static and dynamic libraries + make("lib", "solib") + make("install", "PREFIX=%s" % prefix) From f20247ae55362424d1d5543840a21142017661af Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 10:08:54 +0100 Subject: [PATCH 269/339] environment : refactoreded set_compiler_environment_variables --- lib/spack/spack/build_environment.py | 38 +++++++++++++++------------- lib/spack/spack/environment.py | 17 ++++++++++++- lib/spack/spack/test/database.py | 1 + lib/spack/spack/test/environment.py | 9 +++++++ 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 392ba7ea4d5..e91a8a19977 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,6 +35,7 @@ import spack from llnl.util.filesystem import * +from spack.environment import EnvironmentModifications, apply_environment_modifications from spack.util.environment import * from spack.util.executable import Executable, which @@ -83,30 +84,32 @@ def __call__(self, *args, **kwargs): def set_compiler_environment_variables(pkg): - assert(pkg.spec.concrete) - compiler = pkg.compiler - + assert pkg.spec.concrete # Set compiler variables used by CMake and autotools - assert all(key in pkg.compiler.link_paths - for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + # Populate an object with the list of environment modifications + # and return it + # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. + env = EnvironmentModifications() link_dir = spack.build_env_path - os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc']) - os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx']) - os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77']) - os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc']) - + env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) + env.set_env('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) + env.set_env('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call + compiler = pkg.compiler if compiler.cc: - os.environ['SPACK_CC'] = compiler.cc + env.set_env('SPACK_CC', compiler.cc) if compiler.cxx: - os.environ['SPACK_CXX'] = compiler.cxx + env.set_env('SPACK_CXX', compiler.cxx) if compiler.f77: - os.environ['SPACK_F77'] = compiler.f77 + env.set_env('SPACK_F77', compiler.f77) if compiler.fc: - os.environ['SPACK_FC'] = compiler.fc + env.set_env('SPACK_FC', compiler.fc) - os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + env.set_env('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + return env def set_build_environment_variables(pkg): @@ -264,9 +267,10 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) + env = EnvironmentModifications() + env.extend(set_compiler_environment_variables(pkg)) + apply_environment_modifications(env) set_build_environment_variables(pkg) - # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 7d3d7af0de1..3b73f1c7a2a 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -73,12 +73,24 @@ class EnvironmentModifications(object): Keeps track of requests to modify the current environment """ - def __init__(self): + def __init__(self, other=None): self.env_modifications = [] + if other is not None: + self._check_other(other) + self.env_modifications.extend(other.env_modifications) def __iter__(self): return iter(self.env_modifications) + def extend(self, other): + self._check_other(other) + self.env_modifications.extend(other.env_modifications) + + @staticmethod + def _check_other(other): + if not isinstance(other, EnvironmentModifications): + raise TypeError('other must be an instance of EnvironmentModifications') + def set_env(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -138,6 +150,9 @@ def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: modifications[item.name].append(item) + # TODO : once we organized the modifications into a dictionary that maps an environment variable + # TODO : to a list of action to be done on it, we may easily spot inconsistencies and warn the user if + # TODO : something suspicious is happening return modifications diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 9a57e1f03ea..ce6e8a0552b 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,6 +26,7 @@ These tests check the database is functioning properly, both in memory and in its file """ +import os.path import multiprocessing import shutil import tempfile diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index dff3863d322..17061c8fd0e 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -48,3 +48,12 @@ def test_path_manipulation(self): self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) + + def test_extra_arguments(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value', who='Pkg1') + apply_environment_modifications(env) + self.assertEqual('dummy value', os.environ['A']) + + def test_copy(self): + pass From bcea1df01ce145aea5e49eaaceb7a063b44ddf80 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 10:49:33 +0100 Subject: [PATCH 270/339] environment : refactoreded set_build_environment_variables --- lib/spack/spack/build_environment.py | 52 ++++++++++--------- lib/spack/spack/environment.py | 3 ++ .../builtin/packages/mvapich2/package.py | 2 +- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e91a8a19977..770e191ac90 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,7 +35,7 @@ import spack from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, apply_environment_modifications +from spack.environment import EnvironmentModifications, apply_environment_modifications, concatenate_paths from spack.util.environment import * from spack.util.executable import Executable, which @@ -127,44 +127,45 @@ def set_build_environment_variables(pkg): # handled by putting one in the /case-insensitive # directory. Add that to the path too. env_paths = [] - def add_env_path(path): - env_paths.append(path) - ci = join_path(path, 'case-insensitive') - if os.path.isdir(ci): env_paths.append(ci) - add_env_path(spack.build_env_path) - add_env_path(join_path(spack.build_env_path, pkg.compiler.name)) + for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + env_paths.append(item) + ci = join_path(item, 'case-insensitive') + if os.path.isdir(ci): + env_paths.append(ci) - path_put_first("PATH", env_paths) - path_set(SPACK_ENV_PATH, env_paths) + env = EnvironmentModifications() + for item in reversed(env_paths): + env.prepend_path('PATH', item) + env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) - # Prefixes of all of the package's dependencies go in - # SPACK_DEPENDENCIES + # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] - path_set(SPACK_DEPENDENCIES, dep_prefixes) + env.set_env(SPACK_DEPENDENCIES, concatenate_paths(dep_prefixes)) + env.set_env('CMAKE_PREFIX_PATH', concatenate_paths(dep_prefixes)) # Add dependencies to CMAKE_PREFIX_PATH # Install prefix - os.environ[SPACK_PREFIX] = pkg.prefix + env.set_env(SPACK_PREFIX, pkg.prefix) # Install root prefix - os.environ[SPACK_INSTALL] = spack.install_path + env.set_env(SPACK_INSTALL, spack.install_path) # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") + env.unset_env('LD_LIBRARY_PATH') + env.unset_env('LD_RUN_PATH') + env.unset_env('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes] - path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)]) + bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + for item in bin_dirs: + env.prepend_path('PATH', item) # Working directory for the spack command itself, for debug logs. if spack.debug: - os.environ[SPACK_DEBUG] = "TRUE" - os.environ[SPACK_SHORT_SPEC] = pkg.spec.short_spec - os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir - - # Add dependencies to CMAKE_PREFIX_PATH - path_set("CMAKE_PREFIX_PATH", dep_prefixes) + env.set_env(SPACK_DEBUG, 'TRUE') + env.set_env(SPACK_SHORT_SPEC, pkg.spec.short_spec) + env.set_env(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] @@ -173,8 +174,9 @@ def add_env_path(path): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + env.set_env('PKG_CONFIG_PATH', concatenate_paths(pkg_config_dirs)) + return env def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -269,8 +271,8 @@ def setup_package(pkg): """Execute all environment setup routines.""" env = EnvironmentModifications() env.extend(set_compiler_environment_variables(pkg)) + env.extend(set_build_environment_variables(pkg)) apply_environment_modifications(env) - set_build_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 3b73f1c7a2a..18fef1ef12c 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -146,6 +146,9 @@ def remove_path(self, name, path, **kwargs): self.env_modifications.append(item) +def concatenate_paths(paths): + return ':'.join(str(item) for item in paths) + def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index af5ed1b0888..e4e95f92af7 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -123,7 +123,7 @@ def set_network_type(self, spec, configure_args): count += 1 if count > 1: raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') - + network_options = [] # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] From c85888eb5763d453af7b7255b6d8c3461082362f Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 13:36:41 +0100 Subject: [PATCH 271/339] package : added `environment_modifications` --- lib/spack/spack/build_environment.py | 7 +++--- lib/spack/spack/package.py | 4 ++++ .../repos/builtin/packages/mpich/package.py | 16 +++++++++----- .../packages/netlib-scalapack/package.py | 3 +-- .../repos/builtin/packages/openmpi/package.py | 14 +++++++----- .../repos/builtin/packages/python/package.py | 20 +++++++++-------- .../repos/builtin/packages/qt/package.py | 12 +++++----- .../repos/builtin/packages/ruby/package.py | 22 +++++++++++-------- 8 files changed, 58 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 770e191ac90..d321a0e495e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -178,6 +178,7 @@ def set_build_environment_variables(pkg): return env + def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. @@ -272,7 +273,6 @@ def setup_package(pkg): env = EnvironmentModifications() env.extend(set_compiler_environment_variables(pkg)) env.extend(set_build_environment_variables(pkg)) - apply_environment_modifications(env) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, @@ -284,8 +284,9 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): - dep_spec.package.setup_dependent_environment( - pkg.module, dep_spec, pkg.spec) + env.extend(dep_spec.package.environment_modifications(pkg.module, dep_spec, pkg.spec)) + dep_spec.package.setup_dependent_environment(pkg.module, dep_spec, pkg.spec) + apply_environment_modifications(env) def fork(pkg, function): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf8965..224d7f8f4b9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -63,6 +63,7 @@ import spack.url import spack.util.web import spack.fetch_strategy as fs +from spack.environment import EnvironmentModifications from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive, extension @@ -983,6 +984,9 @@ def module(self): fromlist=[self.__class__.__name__]) + def environment_modifications(self, module, spec, dependent_spec): + return EnvironmentModifications() + def setup_dependent_environment(self, module, spec, dependent_spec): """Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index e2b3654c19a..c85e8febf39 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -46,14 +46,18 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') + def environment_modifications(self, module, spec, dependent_spec): + env = super(Mpich, self).environment_modifications(module, spec, dependent_spec) + env.set_env('MPICH_CC', os.environ['CC']) + env.set_env('MPICH_CXX', os.environ['CXX']) + env.set_env('MPICH_F77', os.environ['F77']) + env.set_env('MPICH_F90', os.environ['FC']) + env.set_env('MPICH_FC', os.environ['FC']) + return env + def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = os.environ['CC'] - os.environ['MPICH_CXX'] = os.environ['CXX'] - os.environ['MPICH_F77'] = os.environ['F77'] - os.environ['MPICH_F90'] = os.environ['FC'] - os.environ['MPICH_FC'] = os.environ['FC'] - + # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 22d538560e8..6dbf3674756 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -46,5 +46,4 @@ def setup_dependent_environment(self, module, spec, dependent_spec): spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib spec['scalapack'].cc_link = spec['scalapack'].fc_link - spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, - 'libscalapack%s' % lib_suffix)] + spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, 'libscalapack%s' % lib_suffix)] diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index e4484af8c5e..83a3fe7a4f3 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,12 +41,14 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' - os.environ['OMPI_CXX'] = 'c++' - os.environ['OMPI_FC'] = 'f90' - os.environ['OMPI_F77'] = 'f77' + def environment_modifications(self, module, spec, dependent_spec): + env = super(Openmpi, self).environment_modifications(module, spec, dependent_spec) + # FIXME : the compilers should point to the current wrappers, not to generic cc etc. + env.set_env('OMPI_CC', 'cc') + env.set_env('OMPI_CXX', 'c++') + env.set_env('OMPI_FC', 'f90') + env.set_env('OMPI_F77', 'f77') + return env def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index dd240d1ea0a..39ced0a1200 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -90,6 +90,17 @@ def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') + def environment_modifications(self, module, spec, dependent_spec): + env = super(Python, self).environment_modifications(module, spec, dependent_spec) + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) + + def setup_dependent_environment(self, module, spec, ext_spec): """Called before python modules' install() methods. @@ -111,15 +122,6 @@ def setup_dependent_environment(self, module, spec, ext_spec): # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) - - # ======================================================================== # Handle specifics of activating and deactivating python modules. # ======================================================================== diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 91afa420c17..373623cd959 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -52,11 +52,13 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - - def setup_dependent_environment(self, module, spec, dep_spec): - """Dependencies of Qt find it using the QTDIR environment variable.""" - os.environ['QTDIR'] = self.prefix - + def environment_modifications(self, module, spec, dep_spec): + """ + Dependencies of Qt find it using the QTDIR environment variable + """ + env = super(Qt, self).environment_modifications(module, spec, dep_spec) + env.set_env['QTDIR'] = self.prefix + return env def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 6b6242362cd..9ec0afa2683 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -15,10 +15,21 @@ class Ruby(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") + def environment_modifications(self, module, spec, ext_spec): + env = super(Ruby, self).environment_modifications(module, spec, ext_spec) + # Set GEM_PATH to include dependent gem directories + ruby_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + ruby_paths.append(d.prefix) + env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + # The actual installation path for this gem + env.set_env('GEM_HOME', ext_spec.prefix) + return env + def setup_dependent_environment(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. @@ -31,11 +42,4 @@ def setup_dependent_environment(self, module, spec, ext_spec): module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - # Set GEM_PATH to include dependent gem directories - ruby_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - ruby_paths.append(d.prefix) - os.environ['GEM_PATH'] = ':'.join(ruby_paths) - # The actual installation path for this gem - os.environ['GEM_HOME'] = ext_spec.prefix + From 572cb93bf8131d222d2d08bca13fd9de6fded1f4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 14:05:30 +0100 Subject: [PATCH 272/339] package : renamed `setup_dependent_environment` to `module_modifications` --- lib/spack/spack/build_environment.py | 4 ++-- lib/spack/spack/package.py | 4 ++-- var/spack/repos/builtin/packages/mpich/package.py | 6 +++--- .../builtin/packages/netlib-scalapack/package.py | 2 +- var/spack/repos/builtin/packages/openmpi/package.py | 4 ++-- var/spack/repos/builtin/packages/python/package.py | 10 ++++------ var/spack/repos/builtin/packages/qt/package.py | 7 ++----- var/spack/repos/builtin/packages/ruby/package.py | 13 ++++++------- 8 files changed, 22 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d321a0e495e..e86a7c413a9 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -284,8 +284,8 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): - env.extend(dep_spec.package.environment_modifications(pkg.module, dep_spec, pkg.spec)) - dep_spec.package.setup_dependent_environment(pkg.module, dep_spec, pkg.spec) + dep_spec.package.module_modifications(pkg.module, dep_spec, pkg.spec) + env.extend(dep_spec.package.environment_modifications(pkg.spec)) apply_environment_modifications(env) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 224d7f8f4b9..c1a5c912be7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -984,10 +984,10 @@ def module(self): fromlist=[self.__class__.__name__]) - def environment_modifications(self, module, spec, dependent_spec): + def environment_modifications(self, dependent_spec): return EnvironmentModifications() - def setup_dependent_environment(self, module, spec, dependent_spec): + def module_modifications(self, module, spec, dependent_spec): """Called before the install() method of dependents. Default implementation does nothing, but this can be diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c85e8febf39..d298981c925 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -46,8 +46,8 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def environment_modifications(self, module, spec, dependent_spec): - env = super(Mpich, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, dependent_spec): + env = super(Mpich, self).environment_modifications(dependent_spec) env.set_env('MPICH_CC', os.environ['CC']) env.set_env('MPICH_CXX', os.environ['CXX']) env.set_env('MPICH_F77', os.environ['F77']) @@ -55,7 +55,7 @@ def environment_modifications(self, module, spec, dependent_spec): env.set_env('MPICH_FC', os.environ['FC']) return env - def setup_dependent_environment(self, module, spec, dep_spec): + def module_modifications(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 6dbf3674756..ecdea464420 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_environment(self, module, spec, dependent_spec): + def module_modifications(self, module, spec, dependent_spec): # TODO treat OS that are not Linux... lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 83a3fe7a4f3..3a141704576 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,8 +41,8 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def environment_modifications(self, module, spec, dependent_spec): - env = super(Openmpi, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, dependent_spec): + env = super(Openmpi, self).environment_modifications(dependent_spec) # FIXME : the compilers should point to the current wrappers, not to generic cc etc. env.set_env('OMPI_CC', 'cc') env.set_env('OMPI_CXX', 'c++') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 39ced0a1200..307cec726b6 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -89,19 +89,17 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - - def environment_modifications(self, module, spec, dependent_spec): - env = super(Python, self).environment_modifications(module, spec, dependent_spec) + def environment_modifications(self, extension_spec): + env = super(Python, self).environment_modifications(extension_spec) # Set PYTHONPATH to include site-packages dir for the # extension and any other python extensions it depends on. python_paths = [] - for d in ext_spec.traverse(): + for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) env.set_env['PYTHONPATH'] = ':'.join(python_paths) - - def setup_dependent_environment(self, module, spec, ext_spec): + def module_modifications(self, module, spec, ext_spec): """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 373623cd959..8391ded3e0b 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -52,11 +52,8 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def environment_modifications(self, module, spec, dep_spec): - """ - Dependencies of Qt find it using the QTDIR environment variable - """ - env = super(Qt, self).environment_modifications(module, spec, dep_spec) + def environment_modifications(self, dependent_spec): + env = super(Qt, self).environment_modifications(dependent_spec) env.set_env['QTDIR'] = self.prefix return env diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 9ec0afa2683..9caea30ef46 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -1,6 +1,5 @@ from spack import * -import spack -import os + class Ruby(Package): """A dynamic, open source programming language with a focus on @@ -18,19 +17,19 @@ def install(self, spec, prefix): make() make("install") - def environment_modifications(self, module, spec, ext_spec): - env = super(Ruby, self).environment_modifications(module, spec, ext_spec) + def environment_modifications(self, extension_spec): + env = super(Ruby, self).environment_modifications(extension_spec) # Set GEM_PATH to include dependent gem directories ruby_paths = [] - for d in ext_spec.traverse(): + for d in extension_spec.traverse(): if d.package.extends(self.spec): ruby_paths.append(d.prefix) env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem - env.set_env('GEM_HOME', ext_spec.prefix) + env.set_env('GEM_HOME', extension_spec.prefix) return env - def setup_dependent_environment(self, module, spec, ext_spec): + def module_modifications(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. From cc3d9f4eb751fe028db5a074fc7bc52fbe0a962a Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 15:09:35 +0100 Subject: [PATCH 273/339] environment : added test, modified docs --- lib/spack/spack/environment.py | 61 ++++++++++++++++++++--------- lib/spack/spack/package.py | 45 +++++++++++++-------- lib/spack/spack/test/environment.py | 10 ++++- lib/spack/spack/util/environment.py | 8 +--- 4 files changed, 80 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 18fef1ef12c..5a68e10c997 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,33 +3,40 @@ import collections -class SetEnv(object): - def __init__(self, name, value, **kwargs): - self.name = name - self.value = value +class AttributeHolder(object): + """ + Policy that permits to store any kind of attribute on self. The attributes must be passed as key/value pairs + during the initialization of the instance. + """ + def __init__(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) + +class SetEnv(AttributeHolder): + def __init__(self, name, value, **kwargs): + super(SetEnv, self).__init__(**kwargs) + self.name = name + self.value = value + def execute(self): os.environ[self.name] = str(self.value) -class UnsetEnv(object): +class UnsetEnv(AttributeHolder): def __init__(self, name, **kwargs): + super(UnsetEnv, self).__init__(**kwargs) self.name = name - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set -class AppendPath(object): +class AppendPath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(AppendPath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -39,12 +46,11 @@ def execute(self): os.environ[self.name] = ':'.join(directories) -class PrependPath(object): +class PrependPath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(PrependPath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -54,12 +60,11 @@ def execute(self): os.environ[self.name] = ':'.join(directories) -class RemovePath(object): +class RemovePath(AttributeHolder): def __init__(self, name, path, **kwargs): + super(RemovePath, self).__init__(**kwargs) self.name = name self.path = path - for key, value in kwargs.items(): - setattr(self, key, value) def execute(self): environment_value = os.environ.get(self.name, '') @@ -70,18 +75,26 @@ def execute(self): class EnvironmentModifications(object): """ - Keeps track of requests to modify the current environment + Keeps track of requests to modify the current environment. """ def __init__(self, other=None): + """ + Initializes a new instance, copying commands from other if it is not None + + Args: + other: another instance of EnvironmentModifications from which (optional) + """ self.env_modifications = [] if other is not None: - self._check_other(other) - self.env_modifications.extend(other.env_modifications) + self.extend(other) def __iter__(self): return iter(self.env_modifications) + def __len__(self): + return len(self.env_modifications) + def extend(self, other): self._check_other(other) self.env_modifications.extend(other.env_modifications) @@ -147,8 +160,18 @@ def remove_path(self, name, path, **kwargs): def concatenate_paths(paths): + """ + Concatenates an iterable of paths into a column separated string + + Args: + paths: iterable of paths + + Returns: + column separated string + """ return ':'.join(str(item) for item in paths) + def validate_environment_modifications(env): modifications = collections.defaultdict(list) for item in env: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c1a5c912be7..5d8258d4cf0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -985,30 +985,41 @@ def module(self): def environment_modifications(self, dependent_spec): + """ + Called before the install() method of dependents. + + Return the list of environment modifications needed by dependents (or extensions). Default implementation does + nothing, but this can be overridden by an extendable package to set up the install environment for its + extensions. This is useful if there are some common steps to installing all extensions for a certain package. + + Example : + + 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could set that variable. + + 2. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + + Args: + dependent_spec: dependent (or extension) of this spec + + Returns: + instance of environment modifications + """ return EnvironmentModifications() def module_modifications(self, module, spec, dependent_spec): - """Called before the install() method of dependents. + """ + Called before the install() method of dependents. - Default implementation does nothing, but this can be - overridden by an extendable package to set up the install - environment for its extensions. This is useful if there are - some common steps to installing all extensions for a + Default implementation does nothing, but this can be overridden by an extendable package to set up the module of + its extensions. This is useful if there are some common steps to installing all extensions for a certain package. - Some examples: - - 1. Installing python modules generally requires PYTHONPATH to - point to the lib/pythonX.Y/site-packages directory in the - module's install prefix. This could set that variable. - - 2. Extensions often need to invoke the 'python' interpreter - from the Python installation being extended. This routine can - put a 'python' Execuable object in the module scope for the - extension package to simplify extension installs. - - 3. A lot of Qt extensions need QTDIR set. This can be used to do that. + Example : + 1. Extensions often need to invoke the 'python' interpreter from the Python installation being extended. + This routine can put a 'python' Executable object in the module scope for the extension package to simplify + extension installs. """ pass diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 17061c8fd0e..97581ecb76a 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -55,5 +55,11 @@ def test_extra_arguments(self): apply_environment_modifications(env) self.assertEqual('dummy value', os.environ['A']) - def test_copy(self): - pass + def test_extend(self): + env = EnvironmentModifications() + env.set_env('A', 'dummy value') + env.set_env('B', 3) + copy_construct = EnvironmentModifications(env) + self.assertEqual(len(copy_construct), 2) + for x, y in zip(env, copy_construct): + self.assertIs(x, y) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index ae8e5708bef..00cda8d5084 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -40,11 +40,13 @@ def env_flag(name): return False +# FIXME : remove this function ? def path_set(var_name, directories): path_str = ":".join(str(dir) for dir in directories) os.environ[var_name] = path_str +# FIXME : remove this function ? def path_put_first(var_name, directories): """Puts the provided directories first in the path, adding them if they're not already there. @@ -59,12 +61,6 @@ def path_put_first(var_name, directories): path_set(var_name, new_path) -def pop_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - dictionary.pop(key) - - def dump_environment(path): """Dump the current environment out to a file.""" with open(path, 'w') as env_file: From ccced9f290bc213561209db2479bccfb5605366d Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 15:12:39 +0100 Subject: [PATCH 274/339] package : optimized imports --- lib/spack/spack/package.py | 41 ++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5d8258d4cf0..042833964ac 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -34,41 +34,34 @@ README. """ import os -import errno import re -import shutil -import time -import itertools -import subprocess -import platform as py_platform -import multiprocessing -from urlparse import urlparse, urljoin import textwrap -from StringIO import StringIO +import time import llnl.util.tty as tty -from llnl.util.tty.log import log_output -from llnl.util.link_tree import LinkTree -from llnl.util.filesystem import * -from llnl.util.lang import * - import spack -import spack.error -import spack.compilers -import spack.mirror -import spack.hooks -import spack.directives -import spack.repository import spack.build_environment +import spack.compilers +import spack.directives +import spack.error +import spack.fetch_strategy as fs +import spack.hooks +import spack.mirror +import spack.repository import spack.url import spack.util.web -import spack.fetch_strategy as fs +from StringIO import StringIO +from llnl.util.filesystem import * +from llnl.util.lang import * +from llnl.util.link_tree import LinkTree +from llnl.util.tty.log import log_output from spack.environment import EnvironmentModifications -from spack.version import * from spack.stage import Stage, ResourceStage, StageComposite -from spack.util.compression import allowed_archive, extension -from spack.util.executable import ProcessError +from spack.util.compression import allowed_archive from spack.util.environment import dump_environment +from spack.util.executable import ProcessError +from spack.version import * +from urlparse import urlparse """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] From c8cc6f4fc111d5dd2d55295e569a10cd5739ceee Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 15 Mar 2016 16:59:29 +0100 Subject: [PATCH 275/339] test : fix for python 2.6 --- lib/spack/spack/test/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 97581ecb76a..f44282d6f39 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -62,4 +62,4 @@ def test_extend(self): copy_construct = EnvironmentModifications(env) self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): - self.assertIs(x, y) + assert x is y From d8a402ae5bcfbea627967531f7ec35cf77946b4a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Mar 2016 13:33:56 -0800 Subject: [PATCH 276/339] Add compiler info and a simple libdwarf build to the checks. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ab379be4867..1bed6b08740 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,10 @@ before_install: script: - . share/spack/setup-env.sh + - spack compilers + - spack config get compilers - spack test + - spack install -v libdwarf notifications: email: From 15bbd088e6007a9a6df8c4427340a371e5871506 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Mar 2016 14:38:06 -0700 Subject: [PATCH 277/339] Fix #551: version bug in `spack create` - `spack create` now sets a proper version in generated file, based on the filename, even if it can't find any tarballs for the package. --- lib/spack/spack/cmd/create.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 4564143f834..f0cd50b8df6 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -208,7 +208,7 @@ def find_repository(spec, args): return repo -def fetch_tarballs(url, name, args): +def fetch_tarballs(url, name, version): """Try to find versions of the supplied archive by scraping the web. Prompts the user to select how many to download if many are found. @@ -222,7 +222,7 @@ def fetch_tarballs(url, name, args): archives_to_fetch = 1 if not versions: # If the fetch failed for some reason, revert to what the user provided - versions = { "version" : url } + versions = { version : url } elif len(versions) > 1: tty.msg("Found %s versions of %s:" % (len(versions), name), *spack.cmd.elide_list( @@ -256,7 +256,7 @@ def create(parser, args): tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) - versions, urls = fetch_tarballs(url, name, args) + versions, urls = fetch_tarballs(url, name, version) # Try to guess what configure system is used. guesser = ConfigureGuesser() From 7eca383b10f017666c320ffd7deb6cae83b5c26b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 10 Mar 2016 01:12:25 -0800 Subject: [PATCH 278/339] Add sanity check paths to packages; fix #505 --- lib/spack/spack/package.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 696adaf8965..02fb3e58348 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,6 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False + """List of prefix-relative file paths. If these do not exist after + install, or if they exist but are not files, sanity checks fail. + """ + sanity_check_files = [] + + """List of prefix-relative directory paths. If these do not exist + after install, or if they exist but are not directories, sanity + checks will fail. + """ + sanity_check_dirs = [] + def __init__(self, spec): # this determines how the package should be built. @@ -909,7 +920,7 @@ def build_process(): raise e # Ensure that something was actually installed. - self._sanity_check_install() + self.sanity_check_prefix() # Copy provenance into the install directory on success log_install_path = spack.install_layout.build_log_path(self.spec) @@ -952,7 +963,18 @@ def build_process(): spack.hooks.post_install(self) - def _sanity_check_install(self): + def sanity_check_prefix(self): + """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): + for path in path_list: + abs_path = os.path.join(self.prefix, path) + if not predicate(abs_path): + raise InstallError("Install failed for %s. No such %s in prefix: %s" + % (self.name, filetype, path)) + + check_paths(self.sanity_check_files, 'file', os.path.isfile) + check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) if not installed: From 68d22253eca40bc4223234d6c26204f292be6a0c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Mar 2016 17:22:48 -0700 Subject: [PATCH 279/339] Add sanity checks to the libelf build. --- var/spack/repos/builtin/packages/libelf/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 29bc21b65c4..0fcb56c164d 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,6 +38,9 @@ class Libelf(Package): provides('elf') + sanity_check_files = ['include/libelf.h'] + sanity_check_dirs = ['lib'] + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", From b45ec3f04e3627dbe3633239560873ae01bf3beb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 10:55:28 +0100 Subject: [PATCH 280/339] environment : simplified modification of the environment --- lib/spack/spack/build_environment.py | 12 ++-- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/environment.py | 58 ++++++++++--------- lib/spack/spack/test/environment.py | 12 ++-- lib/spack/spack/util/environment.py | 2 +- .../repos/builtin/packages/python/package.py | 1 + 6 files changed, 47 insertions(+), 40 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e86a7c413a9..e5d256a2e01 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -35,7 +35,7 @@ import spack from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, apply_environment_modifications, concatenate_paths +from spack.environment import EnvironmentModifications, concatenate_paths from spack.util.environment import * from spack.util.executable import Executable, which @@ -283,10 +283,12 @@ def setup_package(pkg): set_module_variables_for_package(pkg, mod) # Allow dependencies to set up environment as well. - for dep_spec in pkg.spec.traverse(root=False): - dep_spec.package.module_modifications(pkg.module, dep_spec, pkg.spec) - env.extend(dep_spec.package.environment_modifications(pkg.spec)) - apply_environment_modifications(env) + for dependency_spec in pkg.spec.traverse(root=False): + dependency_spec.package.module_modifications(pkg.module, dependency_spec, pkg.spec) + env.extend(dependency_spec.package.environment_modifications(pkg.spec)) + # TODO : implement validation + #validate(env) + env.apply_modifications() def fork(pkg, function): diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 1d6867c1d96..315d9fc9266 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -80,7 +80,7 @@ def module_find(mtype, spec_array): if not os.path.isfile(mod.file_name): tty.die("No %s module is installed for %s" % (mtype, spec)) - print mod.use_name + print(mod.use_name) def module_refresh(): diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 5a68e10c997..b557bf0bbcf 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -158,41 +158,43 @@ def remove_path(self, name, path, **kwargs): item = RemovePath(name, path, **kwargs) self.env_modifications.append(item) + def group_by_name(self): + """ + Returns a dict of the modifications grouped by variable name + + Returns: + dict mapping the environment variable name to the modifications to be done on it + """ + modifications = collections.defaultdict(list) + for item in self: + modifications[item.name].append(item) + return modifications + + def clear(self): + """ + Clears the current list of modifications + """ + self.env_modifications.clear() + + def apply_modifications(self): + """ + Applies the modifications and clears the list + """ + modifications = self.group_by_name() + # Apply the modifications to the environment variables one variable at a time + for name, actions in sorted(modifications.items()): + for x in actions: + x.execute() + def concatenate_paths(paths): """ - Concatenates an iterable of paths into a column separated string + Concatenates an iterable of paths into a string of column separated paths Args: paths: iterable of paths Returns: - column separated string + string """ return ':'.join(str(item) for item in paths) - - -def validate_environment_modifications(env): - modifications = collections.defaultdict(list) - for item in env: - modifications[item.name].append(item) - # TODO : once we organized the modifications into a dictionary that maps an environment variable - # TODO : to a list of action to be done on it, we may easily spot inconsistencies and warn the user if - # TODO : something suspicious is happening - return modifications - - -def apply_environment_modifications(env): - """ - Modifies the current environment according to the request in env - - Args: - env: object storing modifications to the environment - """ - modifications = validate_environment_modifications(env) - - # Cycle over the environment variables that will be modified - for variable, actions in modifications.items(): - # Execute all the actions in the order they were issued - for x in actions: - x.execute() diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index f44282d6f39..d0b093b0543 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -1,6 +1,6 @@ import unittest import os -from spack.environment import EnvironmentModifications, apply_environment_modifications +from spack.environment import EnvironmentModifications class EnvironmentTest(unittest.TestCase): @@ -15,7 +15,7 @@ def test_set_env(self): env = EnvironmentModifications() env.set_env('A', 'dummy value') env.set_env('B', 3) - apply_environment_modifications(env) + env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) self.assertEqual(str(3), os.environ['B']) @@ -23,7 +23,7 @@ def test_unset_env(self): env = EnvironmentModifications() self.assertEqual('foo', os.environ['UNSET_ME']) env.unset_env('UNSET_ME') - apply_environment_modifications(env) + env.apply_modifications() self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') def test_path_manipulation(self): @@ -43,7 +43,7 @@ def test_path_manipulation(self): env.remove_path('REMOVE_PATH_LIST', '/remove/this') env.remove_path('REMOVE_PATH_LIST', '/duplicate/') - apply_environment_modifications(env) + env.apply_modifications() self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) @@ -52,7 +52,9 @@ def test_path_manipulation(self): def test_extra_arguments(self): env = EnvironmentModifications() env.set_env('A', 'dummy value', who='Pkg1') - apply_environment_modifications(env) + for x in env: + assert hasattr(x, 'who') + env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) def test_extend(self): diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 00cda8d5084..1485992b0f1 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -64,5 +64,5 @@ def path_put_first(var_name, directories): def dump_environment(path): """Dump the current environment out to a file.""" with open(path, 'w') as env_file: - for key,val in sorted(os.environ.items()): + for key, val in sorted(os.environ.items()): env_file.write("%s=%s\n" % (key, val)) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 307cec726b6..2f9948d451c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -98,6 +98,7 @@ def environment_modifications(self, extension_spec): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) env.set_env['PYTHONPATH'] = ':'.join(python_paths) + return env def module_modifications(self, module, spec, ext_spec): """Called before python modules' install() methods. From 9c2996667433e4f69eb25cce2d74a94caeb1646c Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 16 Mar 2016 12:57:17 +0100 Subject: [PATCH 281/339] Change urls in binutils to help 'spack checksum/versions' + adding latest version --- var/spack/repos/builtin/packages/binutils/package.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index de04221e33a..897539a4391 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -4,10 +4,13 @@ class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" homepage = "http://www.gnu.org/software/binutils/" - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2") - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2") - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2") - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2") + url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + + version('2.26', '64146a0faa3b411ba774f47d41de239f') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") From 597727f8bedc894330dfd26eab1a82859980f2f1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 15:19:13 +0100 Subject: [PATCH 282/339] tclmodules : added hooks to process EnvironmentModifications objects --- lib/spack/spack/modules.py | 143 +++++++++++++----- .../repos/builtin/packages/mpich/package.py | 25 ++- .../repos/builtin/packages/python/package.py | 15 +- 3 files changed, 133 insertions(+), 50 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index c27043db8cd..1a0a0fd4d61 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -47,18 +47,18 @@ __all__ = ['EnvModule', 'Dotkit', 'TclModule'] import os +import os.path import re -import textwrap import shutil +import textwrap from glob import glob import llnl.util.tty as tty +import spack +from spack.environment import * from llnl.util.filesystem import join_path, mkdirp -import spack - -"""Registry of all types of modules. Entries created by EnvModule's - metaclass.""" +# Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} @@ -79,6 +79,32 @@ def print_help(): "") +class PathInspector(object): + dirname2varname = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',), + 'pkgconfig': ('PKG_CONFIG_PATH',) + } + + def __call__(self, env, directory, names): + for name in names: + variables = PathInspector.dirname2varname.get(name, None) + if variables is None: + continue + absolute_path = join_path(os.path.abspath(directory), name) + for variable in variables: + env.prepend_path(variable, absolute_path) + + +def inspect_path(path): + env, inspector = EnvironmentModifications(), PathInspector() + os.path.walk(path, inspector, env) + return env + + class EnvModule(object): name = 'env_module' @@ -88,21 +114,27 @@ def __init__(cls, name, bases, dict): if cls.name != 'env_module': module_types[cls.name] = cls - def __init__(self, spec=None): # category in the modules system # TODO: come up with smarter category names. self.category = "spack" - # Descriptions for the module system's UI - self.short_description = "" - self.long_description = "" - # dict pathname -> list of directories to be prepended to in # the module file. self._paths = None self.spec = spec + self.pkg = spec.package # Just stored for convenience + # short description default is just the package + version + # packages can provide this optional attribute + self.short_description = spec.format("$_ $@") + if hasattr(self.pkg, 'short_description'): + self.short_description = self.pkg.short_description + + # long description is the docstring with reduced whitespace. + self.long_description = None + if self.spec.package.__doc__: + self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) @property def paths(self): @@ -130,26 +162,19 @@ def add_path(path_name, directory): add_path(var, directory) # Add python path unless it's an actual python installation - # TODO: is there a better way to do this? + # TODO : is there a better way to do this? + # FIXME : add PYTHONPATH to every python package if self.spec.name != 'python': site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) if site_packages: add_path('PYTHONPATH', site_packages[0]) + # FIXME : Same for GEM_PATH if self.spec.package.extends(spack.spec.Spec('ruby')): - add_path('GEM_PATH', self.spec.prefix) - - # short description is just the package + version - # TODO: maybe packages can optionally provide it. - self.short_description = self.spec.format("$_ $@") - - # long description is the docstring with reduced whitespace. - if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + add_path('GEM_PATH', self.spec.prefix) return self._paths - def write(self): """Write out a module file for this object.""" module_dir = os.path.dirname(self.file_name) @@ -160,9 +185,18 @@ def write(self): if not self.paths: return - with open(self.file_name, 'w') as f: - self._write(f) + # Construct the changes that needs to be done on the environment for + env = inspect_path(self.spec.prefix) + # FIXME : move the logic to inspection + env.prepend_path('CMAKE_PREFIX_PATH', self.spec.prefix) + # FIXME : decide how to distinguish between calls done in the installation and elsewhere + env.extend(self.spec.package.environment_modifications(None)) + # site_specific = ...` + if not env: + return + with open(self.file_name, 'w') as f: + self._write(f, env) def _write(self, stream): """To be implemented by subclasses.""" @@ -175,14 +209,12 @@ def file_name(self): where this module lives.""" raise NotImplementedError() - @property def use_name(self): """Subclasses should implement this to return the name the module command uses to refer to the package.""" raise NotImplementedError() - def remove(self): mod_file = self.file_name if os.path.exists(mod_file): @@ -205,7 +237,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def _write(self, dk_file): + def _write(self, dk_file, env): # Category if self.category: dk_file.write('#c %s\n' % self.category) @@ -231,6 +263,10 @@ def _write(self, dk_file): class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") + formats = { + PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', + SetEnv: 'setenv {0.name} \"{0.value}\"\n' + } @property def file_name(self): @@ -244,25 +280,56 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) + def process_environment_command(self, env): + for command in env: + # FIXME : how should we handle errors here? + yield self.formats[type(command)].format(command) - def _write(self, m_file): - # TODO: cateogry? - m_file.write('#%Module1.0\n') + def _write(self, module_file, env): + """ + Writes a TCL module file for this package + Args: + module_file: module file stream + env: list of environment modifications to be written in the module file + """ + # TCL Modulefile header + module_file.write('#%Module1.0\n') + # TODO : category ? # Short description if self.short_description: - m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + module_file.write('module-whatis \"%s\"\n\n' % self.short_description) # Long description if self.long_description: - m_file.write('proc ModulesHelp { } {\n') + module_file.write('proc ModulesHelp { } {\n') doc = re.sub(r'"', '\"', self.long_description) - m_file.write("puts stderr \"%s\"\n" % doc) - m_file.write('}\n\n') + module_file.write("puts stderr \"%s\"\n" % doc) + module_file.write('}\n\n') - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) + # Environment modifications + for line in self.process_environment_command(env): + module_file.write(line) - m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) + # FIXME : REMOVE + # def _write(self, m_file): + # # TODO: cateogry? + # m_file.write('#%Module1.0\n') + # + # # Short description + # if self.short_description: + # m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + # + # # Long description + # if self.long_description: + # m_file.write('proc ModulesHelp { } {\n') + # doc = re.sub(r'"', '\"', self.long_description) + # m_file.write("puts stderr \"%s\"\n" % doc) + # m_file.write('}\n\n') + # + # # Path alterations + # for var, dirs in self.paths.items(): + # for directory in dirs: + # m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) + # + # m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index d298981c925..4c34d0308f5 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -25,6 +25,7 @@ from spack import * import os + class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" @@ -48,11 +49,25 @@ class Mpich(Package): def environment_modifications(self, dependent_spec): env = super(Mpich, self).environment_modifications(dependent_spec) - env.set_env('MPICH_CC', os.environ['CC']) - env.set_env('MPICH_CXX', os.environ['CXX']) - env.set_env('MPICH_F77', os.environ['F77']) - env.set_env('MPICH_F90', os.environ['FC']) - env.set_env('MPICH_FC', os.environ['FC']) + + if dependent_spec is None: + # We are not using compiler wrappers + cc = self.compiler.cc + cxx = self.compiler.cxx + f77 = self.compiler.f77 + f90 = fc = self.compiler.fc + else: + # Spack compiler wrappers + cc = os.environ['CC'] + cxx = os.environ['CXX'] + f77 = os.environ['F77'] + f90 = fc = os.environ['FC'] + + env.set_env('MPICH_CC', cc) + env.set_env('MPICH_CXX', cxx) + env.set_env('MPICH_F77', f77) + env.set_env('MPICH_F90', f90) + env.set_env('MPICH_FC', fc) return env def module_modifications(self, module, spec, dep_spec): diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 2f9948d451c..acb36517268 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -91,13 +91,14 @@ def site_packages_dir(self): def environment_modifications(self, extension_spec): env = super(Python, self).environment_modifications(extension_spec) - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in extension_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) + if extension_spec is not None: + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) return env def module_modifications(self, module, spec, ext_spec): From ac762e95a6213e1514f29d3d6501e4a95dd3e1d4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 16 Mar 2016 16:23:02 +0100 Subject: [PATCH 283/339] modules : removed dead code --- lib/spack/spack/modules.py | 205 ++++++++++++------------------------- 1 file changed, 66 insertions(+), 139 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 1a0a0fd4d61..b64a8b32266 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""This module contains code for creating environment modules, which -can include dotkits, tcl modules, lmod, and others. +""" +This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and -removed after an uninstall by post-uninstall hooks. This class -consolidates the logic for creating an abstract description of the -information that module systems need. Currently that includes a -number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. +This class consolidates the logic for creating an abstract description of the information that module systems need. +Currently that includes a number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH @@ -37,26 +35,23 @@ * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH -This module also includes logic for coming up with unique names for -the module files so that they can be found by the various -shell-support files in $SPACK/share/spack/setup-env.*. +This module also includes logic for coming up with unique names for the module files so that they can be found by the +various shell-support files in $SPACK/share/spack/setup-env.*. -Each hook in hooks/ implements the logic for writing its specific type -of module file. +Each hook in hooks/ implements the logic for writing its specific type of module file. """ -__all__ = ['EnvModule', 'Dotkit', 'TclModule'] - import os import os.path import re import shutil import textwrap -from glob import glob import llnl.util.tty as tty import spack -from spack.environment import * from llnl.util.filesystem import join_path, mkdirp +from spack.environment import * + +__all__ = ['EnvModule', 'Dotkit', 'TclModule'] # Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} @@ -79,34 +74,35 @@ def print_help(): "") -class PathInspector(object): - dirname2varname = { - 'bin': ('PATH',), - 'man': ('MANPATH',), - 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'include': ('CPATH',), - 'pkgconfig': ('PKG_CONFIG_PATH',) - } - - def __call__(self, env, directory, names): - for name in names: - variables = PathInspector.dirname2varname.get(name, None) - if variables is None: - continue - absolute_path = join_path(os.path.abspath(directory), name) - for variable in variables: - env.prepend_path(variable, absolute_path) - - def inspect_path(path): + class PathInspector(object): + dirname2varname = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',), + 'pkgconfig': ('PKG_CONFIG_PATH',) + } + + def __call__(self, env, directory, names): + for name in names: + variables = PathInspector.dirname2varname.get(name, None) + if variables is None: + continue + absolute_path = join_path(os.path.abspath(directory), name) + for variable in variables: + env.prepend_path(variable, absolute_path) + env, inspector = EnvironmentModifications(), PathInspector() os.path.walk(path, inspector, env) + env.prepend_path('CMAKE_PREFIX_PATH', path) return env class EnvModule(object): name = 'env_module' + formats = {} class __metaclass__(type): def __init__(cls, name, bases, dict): @@ -119,9 +115,6 @@ def __init__(self, spec=None): # TODO: come up with smarter category names. self.category = "spack" - # dict pathname -> list of directories to be prepended to in - # the module file. - self._paths = None self.spec = spec self.pkg = spec.package # Just stored for convenience @@ -136,44 +129,21 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) - @property - def paths(self): - if self._paths is None: - self._paths = {} - - def add_path(path_name, directory): - path = self._paths.setdefault(path_name, []) - path.append(directory) - - # Add paths if they exist. - for var, directory in [ - ('PATH', self.spec.prefix.bin), - ('MANPATH', self.spec.prefix.man), - ('MANPATH', self.spec.prefix.share_man), - ('LIBRARY_PATH', self.spec.prefix.lib), - ('LIBRARY_PATH', self.spec.prefix.lib64), - ('LD_LIBRARY_PATH', self.spec.prefix.lib), - ('LD_LIBRARY_PATH', self.spec.prefix.lib64), - ('CPATH', self.spec.prefix.include), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: - - if os.path.isdir(directory): - add_path(var, directory) - - # Add python path unless it's an actual python installation - # TODO : is there a better way to do this? - # FIXME : add PYTHONPATH to every python package - if self.spec.name != 'python': - site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - add_path('PYTHONPATH', site_packages[0]) - - # FIXME : Same for GEM_PATH - if self.spec.package.extends(spack.spec.Spec('ruby')): - add_path('GEM_PATH', self.spec.prefix) - - return self._paths + # @property + # def paths(self): + # # Add python path unless it's an actual python installation + # # TODO : is there a better way to do this? + # # FIXME : add PYTHONPATH to every python package + # if self.spec.name != 'python': + # site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + # if site_packages: + # add_path('PYTHONPATH', site_packages[0]) + # + # # FIXME : Same for GEM_PATH + # if self.spec.package.extends(spack.spec.Spec('ruby')): + # add_path('GEM_PATH', self.spec.prefix) + # + # return self._paths def write(self): """Write out a module file for this object.""" @@ -181,14 +151,9 @@ def write(self): if not os.path.exists(module_dir): mkdirp(module_dir) - # If there are no paths, no need for a dotkit. - if not self.paths: - return - - # Construct the changes that needs to be done on the environment for + # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) - # FIXME : move the logic to inspection - env.prepend_path('CMAKE_PREFIX_PATH', self.spec.prefix) + # Package-specific environment modifications # FIXME : decide how to distinguish between calls done in the installation and elsewhere env.extend(self.spec.package.environment_modifications(None)) # site_specific = ...` @@ -196,12 +161,17 @@ def write(self): return with open(self.file_name, 'w') as f: - self._write(f, env) + self.write_header(f) + for line in self.process_environment_command(env): + f.write(line) - def _write(self, stream): - """To be implemented by subclasses.""" + def write_header(self, stream): raise NotImplementedError() + def process_environment_command(self, env): + for command in env: + # FIXME : how should we handle errors here? + yield self.formats[type(command)].format(command) @property def file_name(self): @@ -225,10 +195,14 @@ class Dotkit(EnvModule): name = 'dotkit' path = join_path(spack.share_path, "dotkit") + formats = { + PrependPath: 'dk_alter {0.name} {0.path}\n', + SetEnv: 'dk_setenv {0.name} {0.value}\n' + } + @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, - '%s.dk' % self.use_name) + return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) @property def use_name(self): @@ -237,7 +211,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def _write(self, dk_file, env): + def write_header(self, dk_file): # Category if self.category: dk_file.write('#c %s\n' % self.category) @@ -251,18 +225,11 @@ def _write(self, dk_file, env): for line in textwrap.wrap(self.long_description, 72): dk_file.write("#h %s\n" % line) - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - dk_file.write("dk_alter %s %s\n" % (var, directory)) - - # Let CMake find this package. - dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix) - class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") + formats = { PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', SetEnv: 'setenv {0.name} \"{0.value}\"\n' @@ -272,7 +239,6 @@ class TclModule(EnvModule): def file_name(self): return join_path(TclModule.path, self.spec.architecture, self.use_name) - @property def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, @@ -280,19 +246,7 @@ def use_name(self): self.spec.compiler.version, self.spec.dag_hash()) - def process_environment_command(self, env): - for command in env: - # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(command) - - def _write(self, module_file, env): - """ - Writes a TCL module file for this package - - Args: - module_file: module file stream - env: list of environment modifications to be written in the module file - """ + def write_header(self, module_file): # TCL Modulefile header module_file.write('#%Module1.0\n') # TODO : category ? @@ -306,30 +260,3 @@ def _write(self, module_file, env): doc = re.sub(r'"', '\"', self.long_description) module_file.write("puts stderr \"%s\"\n" % doc) module_file.write('}\n\n') - - # Environment modifications - for line in self.process_environment_command(env): - module_file.write(line) - - # FIXME : REMOVE - # def _write(self, m_file): - # # TODO: cateogry? - # m_file.write('#%Module1.0\n') - # - # # Short description - # if self.short_description: - # m_file.write('module-whatis \"%s\"\n\n' % self.short_description) - # - # # Long description - # if self.long_description: - # m_file.write('proc ModulesHelp { } {\n') - # doc = re.sub(r'"', '\"', self.long_description) - # m_file.write("puts stderr \"%s\"\n" % doc) - # m_file.write('}\n\n') - # - # # Path alterations - # for var, dirs in self.paths.items(): - # for directory in dirs: - # m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) - # - # m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) From bb04d5cc63182b6129f9abf78db260a2dab9f506 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 16 Mar 2016 14:27:22 -0500 Subject: [PATCH 284/339] Fix grammar in mirrors documentation --- lib/spack/docs/mirrors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index b20fedb55f1..dad04d053b0 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -38,7 +38,7 @@ contains tarballs for each package, named after each package. .. note:: - Archives are **not** named exactly they were in the package's fetch + Archives are **not** named exactly the way they were in the package's fetch URL. They have the form ``-.``, where ```` is Spack's name for the package, ```` is the version of the tarball, and ```` is whatever format the From fa2a66db6721ef5d7b54b9cede69aa2f1b5b7531 Mon Sep 17 00:00:00 2001 From: citibeth Date: Wed, 16 Mar 2016 15:46:59 -0400 Subject: [PATCH 285/339] 1. Disabled git versions known to have vulnerabilities. 2. Added autoconf command to allow building directly from GitHub source. --- .../repos/builtin/packages/git/package.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index ddc5078c4d3..586b6ce3c39 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -5,14 +5,22 @@ class Git(Package): system designed to handle everything from small to very large projects with speed and efficiency.""" homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz" + url = "https://github.com/git/git/tarball/v2.7.1" - version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') - version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') - version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') - version('2.6.0', 'eb76a07148d94802a1745d759716a57e') - version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') - version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') + version('2.8.0-rc2', 'c2cf9f2cc70e35f2fafbaf9258f82e4c') + version('2.7.3', 'fa1c008b56618c355a32ba4a678305f6') + version('2.7.1', 'bf0706b433a8dedd27a63a72f9a66060') + + + # See here for info on vulnerable Git versions: + # http://www.theregister.co.uk/2016/03/16/git_server_client_patch_now/ + # All the following are vulnerable + #version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') + #version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') + #version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') + #version('2.6.0', 'eb76a07148d94802a1745d759716a57e') + #version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') + #version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') # Git compiles with curl support by default on but if your system @@ -24,6 +32,7 @@ class Git(Package): variant("expat", default=False, description="Add the internal support of expat for https push") depends_on("openssl") + depends_on("autoconf") depends_on("curl", when="+curl") depends_on("expat", when="+expat") @@ -47,6 +56,7 @@ def install(self, spec, prefix): if '+expat' in spec: configure_args.append("--with-expat=%s" % spec['expat'].prefix) + which('autoreconf')('-i') configure(*configure_args) make() make("install") From 9cdd79e33f8463699fcea0b668eafac1c9fae1d4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 14:14:18 +0100 Subject: [PATCH 286/339] modules : restored previous logic for path inspection --- lib/spack/spack/modules.py | 50 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index b64a8b32266..7e395736e4f 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -74,29 +74,37 @@ def print_help(): "") -def inspect_path(path): - class PathInspector(object): - dirname2varname = { - 'bin': ('PATH',), - 'man': ('MANPATH',), - 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), - 'include': ('CPATH',), - 'pkgconfig': ('PKG_CONFIG_PATH',) - } +def inspect_path(prefix): + """ + Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment + accordingly when an item is found. - def __call__(self, env, directory, names): - for name in names: - variables = PathInspector.dirname2varname.get(name, None) - if variables is None: - continue - absolute_path = join_path(os.path.abspath(directory), name) - for variable in variables: - env.prepend_path(variable, absolute_path) + Args: + prefix: prefix of the installation - env, inspector = EnvironmentModifications(), PathInspector() - os.path.walk(path, inspector, env) - env.prepend_path('CMAKE_PREFIX_PATH', path) + Returns: + instance of EnvironmentModifications containing the requested modifications + """ + env = EnvironmentModifications() + # Inspect the prefix to check for the existence of common directories + prefix_inspections = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',) + } + for attribute, variables in prefix_inspections.items(): + expected = getattr(prefix, attribute) + if os.path.isdir(expected): + for variable in variables: + env.prepend_path(variable, expected) + # PKGCONFIG + for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')): + if os.path.isdir(expected): + env.prepend_path('PKG_CONFIG_PATH', expected) + # CMake related variables + env.prepend_path('CMAKE_PREFIX_PATH', prefix) return env From f0f0663d1b9ece1e2b0b0a8f720b1325eec443bb Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 15:11:39 +0100 Subject: [PATCH 287/339] package : split `environment_modifications` into `setup_environment` and `setup_dependent_environment`. package : renamed `module_modifications` to `modify_module` for consistency --- lib/spack/spack/build_environment.py | 4 +-- lib/spack/spack/modules.py | 5 ++- lib/spack/spack/package.py | 9 +++-- .../repos/builtin/packages/mpich/package.py | 35 +++++++------------ .../packages/netlib-scalapack/package.py | 2 +- .../repos/builtin/packages/openmpi/package.py | 19 +++++----- .../repos/builtin/packages/python/package.py | 25 ++++++------- .../repos/builtin/packages/qt/package.py | 4 +-- .../repos/builtin/packages/ruby/package.py | 6 ++-- 9 files changed, 49 insertions(+), 60 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index e5d256a2e01..68477145fef 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -284,8 +284,8 @@ def setup_package(pkg): # Allow dependencies to set up environment as well. for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.module_modifications(pkg.module, dependency_spec, pkg.spec) - env.extend(dependency_spec.package.environment_modifications(pkg.spec)) + dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + dependency_spec.package.setup_dependent_environment(env, pkg.spec) # TODO : implement validation #validate(env) env.apply_modifications() diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7e395736e4f..09895d8c441 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -162,9 +162,8 @@ def write(self): # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) # Package-specific environment modifications - # FIXME : decide how to distinguish between calls done in the installation and elsewhere - env.extend(self.spec.package.environment_modifications(None)) - # site_specific = ...` + self.spec.package.setup_environment(env) + # TODO : implement site-specific modifications and filters if not env: return diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 042833964ac..8e56dd1f976 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -977,7 +977,7 @@ def module(self): fromlist=[self.__class__.__name__]) - def environment_modifications(self, dependent_spec): + def setup_environment(self, env): """ Called before the install() method of dependents. @@ -998,9 +998,12 @@ def environment_modifications(self, dependent_spec): Returns: instance of environment modifications """ - return EnvironmentModifications() + pass - def module_modifications(self, module, spec, dependent_spec): + def setup_dependent_environment(self, env, dependent_spec): + self.setup_environment(env) + + def modify_module(self, module, spec, dependent_spec): """ Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 4c34d0308f5..5af9b585ea8 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -47,30 +47,21 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def environment_modifications(self, dependent_spec): - env = super(Mpich, self).environment_modifications(dependent_spec) + def setup_environment(self, env): + env.set_env('MPICH_CC', self.compiler.cc) + env.set_env('MPICH_CXX', self.compiler.cxx) + env.set_env('MPICH_F77', self.compiler.f77) + env.set_env('MPICH_F90', self.compiler.fc) + env.set_env('MPICH_FC', self.compiler.fc) - if dependent_spec is None: - # We are not using compiler wrappers - cc = self.compiler.cc - cxx = self.compiler.cxx - f77 = self.compiler.f77 - f90 = fc = self.compiler.fc - else: - # Spack compiler wrappers - cc = os.environ['CC'] - cxx = os.environ['CXX'] - f77 = os.environ['F77'] - f90 = fc = os.environ['FC'] + def setup_dependent_environment(self, env, dependent_spec): + env.set_env('MPICH_CC', spack_cc) + env.set_env('MPICH_CXX', spack_cxx) + env.set_env('MPICH_F77', spack_f77) + env.set_env('MPICH_F90', spack_f90) + env.set_env('MPICH_FC', spack_fc) - env.set_env('MPICH_CC', cc) - env.set_env('MPICH_CXX', cxx) - env.set_env('MPICH_F77', f77) - env.set_env('MPICH_F90', f90) - env.set_env('MPICH_FC', fc) - return env - - def module_modifications(self, module, spec, dep_spec): + def modify_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index ecdea464420..7e7e5b2e2ee 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def module_modifications(self, module, spec, dependent_spec): + def modify_module(self, module, spec, dependent_spec): # TODO treat OS that are not Linux... lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 3a141704576..7783ca87666 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,14 +41,17 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def environment_modifications(self, dependent_spec): - env = super(Openmpi, self).environment_modifications(dependent_spec) - # FIXME : the compilers should point to the current wrappers, not to generic cc etc. - env.set_env('OMPI_CC', 'cc') - env.set_env('OMPI_CXX', 'c++') - env.set_env('OMPI_FC', 'f90') - env.set_env('OMPI_F77', 'f77') - return env + def setup_environment(self, env): + env.set_env('OMPI_CC', self.compiler.cc) + env.set_env('OMPI_CXX', self.compiler.cxx) + env.set_env('OMPI_FC', self.compiler.fc) + env.set_env('OMPI_F77', self.compiler.f77) + + def setup_dependent_environment(self, env, dependent_spec): + env.set_env('OMPI_CC', spack_cc) + env.set_env('OMPI_CXX', spack_cxx) + env.set_env('OMPI_FC', spack_fc) + env.set_env('OMPI_F77', spack_f77) def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index acb36517268..d47c1d1b2f3 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -89,24 +89,21 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - def environment_modifications(self, extension_spec): - env = super(Python, self).environment_modifications(extension_spec) - if extension_spec is not None: - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in extension_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) - return env + def setup_dependent_environment(self, env, extension_spec): + # Set PYTHONPATH to include site-packages dir for the extension and any other python extensions it depends on. + python_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + env.set_env['PYTHONPATH'] = ':'.join(python_paths) - def module_modifications(self, module, spec, ext_spec): - """Called before python modules' install() methods. + def modify_module(self, module, spec, ext_spec): + """ + Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) """ # Python extension builds can have a global python executable function if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 0adf352be21..35b9d684624 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -55,10 +55,8 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def environment_modifications(self, dependent_spec): - env = super(Qt, self).environment_modifications(dependent_spec) + def setup_environment(self, env): env.set_env['QTDIR'] = self.prefix - return env def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 9caea30ef46..2d1da8c9af1 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -17,8 +17,7 @@ def install(self, spec, prefix): make() make("install") - def environment_modifications(self, extension_spec): - env = super(Ruby, self).environment_modifications(extension_spec) + def setup_dependent_environment(self, env, extension_spec): # Set GEM_PATH to include dependent gem directories ruby_paths = [] for d in extension_spec.traverse(): @@ -27,9 +26,8 @@ def environment_modifications(self, extension_spec): env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem env.set_env('GEM_HOME', extension_spec.prefix) - return env - def module_modifications(self, module, spec, ext_spec): + def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. From 3da4d6664bbee38fde6faeffb39f889698ea320c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 15:38:08 +0100 Subject: [PATCH 288/339] environment : simplified hierarchy according to comments in review --- lib/spack/spack/environment.py | 53 +++++++++-------------------- lib/spack/spack/modules.py | 10 +++--- lib/spack/spack/package.py | 1 + lib/spack/spack/test/environment.py | 2 +- lib/spack/spack/util/environment.py | 2 -- 5 files changed, 24 insertions(+), 44 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index b557bf0bbcf..3ee917fcec6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,73 +3,54 @@ import collections -class AttributeHolder(object): - """ - Policy that permits to store any kind of attribute on self. The attributes must be passed as key/value pairs - during the initialization of the instance. - """ - def __init__(self, **kwargs): - for key, value in kwargs.items(): - setattr(self, key, value) +class NameModifier(object): + def __init__(self, name, **kwargs): + self.name = name + self.args = {'name': name} + self.args.update(kwargs) -class SetEnv(AttributeHolder): +class NameValueModifier(object): def __init__(self, name, value, **kwargs): - super(SetEnv, self).__init__(**kwargs) self.name = name self.value = value + self.args = {'name': name, 'value': value} + self.args.update(kwargs) + +class SetEnv(NameValueModifier): def execute(self): os.environ[self.name] = str(self.value) -class UnsetEnv(AttributeHolder): - def __init__(self, name, **kwargs): - super(UnsetEnv, self).__init__(**kwargs) - self.name = name - +class UnsetEnv(NameModifier): def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set -class AppendPath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(AppendPath, self).__init__(**kwargs) - self.name = name - self.path = path - +class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] # TODO : Check if this is a valid directory name - directories.append(os.path.normpath(self.path)) + directories.append(os.path.normpath(self.value)) os.environ[self.name] = ':'.join(directories) -class PrependPath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(PrependPath, self).__init__(**kwargs) - self.name = name - self.path = path - +class PrependPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] # TODO : Check if this is a valid directory name - directories = [os.path.normpath(self.path)] + directories + directories = [os.path.normpath(self.value)] + directories os.environ[self.name] = ':'.join(directories) -class RemovePath(AttributeHolder): - def __init__(self, name, path, **kwargs): - super(RemovePath, self).__init__(**kwargs) - self.name = name - self.path = path - +class RemovePath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.path)] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] os.environ[self.name] = ':'.join(directories) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 09895d8c441..0f826c73634 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -178,7 +178,7 @@ def write_header(self, stream): def process_environment_command(self, env): for command in env: # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(command) + yield self.formats[type(command)].format(**command.args) @property def file_name(self): @@ -203,8 +203,8 @@ class Dotkit(EnvModule): path = join_path(spack.share_path, "dotkit") formats = { - PrependPath: 'dk_alter {0.name} {0.path}\n', - SetEnv: 'dk_setenv {0.name} {0.value}\n' + PrependPath: 'dk_alter {name} {value}\n', + SetEnv: 'dk_setenv {name} {value}\n' } @property @@ -238,8 +238,8 @@ class TclModule(EnvModule): path = join_path(spack.share_path, "modules") formats = { - PrependPath: 'prepend-path {0.name} \"{0.path}\"\n', - SetEnv: 'setenv {0.name} \"{0.value}\"\n' + PrependPath: 'prepend-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n' } @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8e56dd1f976..680b26d69a7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -979,6 +979,7 @@ def module(self): def setup_environment(self, env): """ + Called before the install() method of dependents. Return the list of environment modifications needed by dependents (or extensions). Default implementation does diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index d0b093b0543..3e03760c016 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -53,7 +53,7 @@ def test_extra_arguments(self): env = EnvironmentModifications() env.set_env('A', 'dummy value', who='Pkg1') for x in env: - assert hasattr(x, 'who') + assert 'who' in x.args env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 1485992b0f1..55e653fd2fb 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -40,13 +40,11 @@ def env_flag(name): return False -# FIXME : remove this function ? def path_set(var_name, directories): path_str = ":".join(str(dir) for dir in directories) os.environ[var_name] = path_str -# FIXME : remove this function ? def path_put_first(var_name, directories): """Puts the provided directories first in the path, adding them if they're not already there. From 38c3c84969c25aaee8f643ff9770759c7e6d9c35 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 17:37:33 +0100 Subject: [PATCH 289/339] environment : added caller information --- lib/spack/spack/environment.py | 27 +++++++++++++++++++++++++-- lib/spack/spack/modules.py | 8 ++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 3ee917fcec6..6d214595a3e 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,6 +1,7 @@ import os import os.path import collections +import inspect class NameModifier(object): @@ -32,7 +33,6 @@ class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - # TODO : Check if this is a valid directory name directories.append(os.path.normpath(self.value)) os.environ[self.name] = ':'.join(directories) @@ -41,7 +41,6 @@ class PrependPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split(':') if environment_value else [] - # TODO : Check if this is a valid directory name directories = [os.path.normpath(self.value)] + directories os.environ[self.name] = ':'.join(directories) @@ -57,6 +56,11 @@ def execute(self): class EnvironmentModifications(object): """ Keeps track of requests to modify the current environment. + + Each call to a method to modify the environment stores the extra information on the caller in the request: + - 'filename' : filename of the module where the caller is defined + - 'lineno': line number where the request occurred + - 'context' : line of code that issued the request that failed """ def __init__(self, other=None): @@ -85,6 +89,20 @@ def _check_other(other): if not isinstance(other, EnvironmentModifications): raise TypeError('other must be an instance of EnvironmentModifications') + def _get_outside_caller_attributes(self): + stack = inspect.stack() + try: + _, filename, lineno, _, context, index = stack[2] + context = context[index].strip() + except Exception: + filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' + args = { + 'filename': filename, + 'lineno': lineno, + 'context': context + } + return args + def set_env(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -93,6 +111,7 @@ def set_env(self, name, value, **kwargs): name: name of the environment variable to be set value: value of the environment variable """ + kwargs.update(self._get_outside_caller_attributes()) item = SetEnv(name, value, **kwargs) self.env_modifications.append(item) @@ -103,6 +122,7 @@ def unset_env(self, name, **kwargs): Args: name: name of the environment variable to be set """ + kwargs.update(self._get_outside_caller_attributes()) item = UnsetEnv(name, **kwargs) self.env_modifications.append(item) @@ -114,6 +134,7 @@ def append_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be appended """ + kwargs.update(self._get_outside_caller_attributes()) item = AppendPath(name, path, **kwargs) self.env_modifications.append(item) @@ -125,6 +146,7 @@ def prepend_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be pre-pended """ + kwargs.update(self._get_outside_caller_attributes()) item = PrependPath(name, path, **kwargs) self.env_modifications.append(item) @@ -136,6 +158,7 @@ def remove_path(self, name, path, **kwargs): name: name of the path list in the environment path: path to be removed """ + kwargs.update(self._get_outside_caller_attributes()) item = RemovePath(name, path, **kwargs) self.env_modifications.append(item) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0f826c73634..d192bbe004e 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -177,8 +177,12 @@ def write_header(self, stream): def process_environment_command(self, env): for command in env: - # FIXME : how should we handle errors here? - yield self.formats[type(command)].format(**command.args) + try: + yield self.formats[type(command)].format(**command.args) + except KeyError: + tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) + tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + @property def file_name(self): From e673127263a03e63d6a64840d1071a788f0c099e Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 18:11:23 +0100 Subject: [PATCH 290/339] package : added documentation --- lib/spack/spack/package.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 680b26d69a7..d0b94dbbeb9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -976,32 +976,41 @@ def module(self): return __import__(self.__class__.__module__, fromlist=[self.__class__.__name__]) - def setup_environment(self, env): """ + Appends in `env` the list of environment modifications needed to use this package outside of spack. + Default implementation does nothing, but this can be overridden if the package needs a particular environment. + + Example : + + 1. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + + Args: + env: list of environment modifications to be updated + """ + pass + + def setup_dependent_environment(self, env, dependent_spec): + """ Called before the install() method of dependents. - Return the list of environment modifications needed by dependents (or extensions). Default implementation does - nothing, but this can be overridden by an extendable package to set up the install environment for its - extensions. This is useful if there are some common steps to installing all extensions for a certain package. + Appends in `env` the list of environment modifications needed by dependents (or extensions) during the + installation of a package. The default implementation delegates to `setup_environment`, but can be overridden + if the modifications to the environment happen to be different from the one needed to use the package outside + of spack. + + This is useful if there are some common steps to installing all extensions for a certain package. Example : 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages directory in the module's install prefix. This could set that variable. - 2. A lot of Qt extensions need `QTDIR` set. This can be used to do that. - Args: + env: list of environment modifications to be updated dependent_spec: dependent (or extension) of this spec - - Returns: - instance of environment modifications """ - pass - - def setup_dependent_environment(self, env, dependent_spec): self.setup_environment(env) def modify_module(self, module, spec, dependent_spec): @@ -1020,12 +1029,10 @@ def modify_module(self, module, spec, dependent_spec): """ pass - def install(self, spec, prefix): """Package implementations override this with their own build configuration.""" raise InstallError("Package %s provides no install method!" % self.name) - def do_uninstall(self, force=False): if not self.installed: raise InstallError(str(self.spec) + " is not installed.") From ac394718ec0aa67c468a8529b930eaade0bcbed1 Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 18:22:07 +0100 Subject: [PATCH 291/339] python : implemented possible solution --- lib/spack/spack/package.py | 11 ++++++++++- var/spack/repos/builtin/packages/py-nose/package.py | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index d0b94dbbeb9..a7ab20137ea 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -37,6 +37,7 @@ import re import textwrap import time +import glob import llnl.util.tty as tty import spack @@ -55,7 +56,6 @@ from llnl.util.lang import * from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output -from spack.environment import EnvironmentModifications from spack.stage import Stage, ResourceStage, StageComposite from spack.util.compression import allowed_archive from spack.util.environment import dump_environment @@ -1236,6 +1236,15 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +class PythonExtension(Package): + def setup_dependent_environment(self, env, dependent_spec): + pass + + def setup_environment(self, env): + site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + env.prepend_path('PYTHONPATH', site_packages[0]) + def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index e7c6cf0264b..e817b8eb513 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -1,6 +1,7 @@ from spack import * +from spack.package import PythonExtension -class PyNose(Package): +class PyNose(PythonExtension): """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" From e55880904376847ce767770344059eef1ba7707c Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 17 Mar 2016 19:32:31 +0100 Subject: [PATCH 292/339] python : fixed typo --- var/spack/repos/builtin/packages/python/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d47c1d1b2f3..d46e1068b6c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -95,7 +95,7 @@ def setup_dependent_environment(self, env, extension_spec): for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env['PYTHONPATH'] = ':'.join(python_paths) + env.set_env('PYTHONPATH', ':'.join(python_paths)) def modify_module(self, module, spec, ext_spec): """ From 9e4c757f50ec98e55a74ad13fda70f12ce788891 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:45:57 -0400 Subject: [PATCH 293/339] Update curl to 7.47.1 --- var/spack/repos/builtin/packages/curl/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c71..ab6305fc084 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -7,6 +7,7 @@ class Curl(Package): homepage = "http://curl.haxx.se" url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + version('7.47.1', '9ea3123449439bbd960cd25cf98796fb') version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') version('7.45.0', '62c1a352b28558f25ba6209214beadc8') version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae') From 90268876f7d740e5b7dcfd079168f248a337791a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Mar 2016 18:49:58 -0700 Subject: [PATCH 294/339] Change sanity_check_[file|dir] to sanity_check_is_[file|dir], per #554 - Add documentation as well. --- lib/spack/docs/packaging_guide.rst | 84 +++++++++++++++---- lib/spack/spack/package.py | 22 +++-- .../repos/builtin/packages/libelf/package.py | 3 +- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 169899212db..c1077e44977 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1559,11 +1559,11 @@ you ask for a particular spec. A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer -packages be built with MVAPICH and GCC. +packages be built with MVAPICH and GCC. Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. -The preferred configuration can be controlled via the +The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the ``etc/spack/packages.yaml`` site configuration. @@ -1582,32 +1582,32 @@ Here's an example packages.yaml file that sets preferred packages: compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] providers: mpi: [mvapich, mpich, openmpi] - + At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and +concretized. The dyninst package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). -These options are used to fill in implicit defaults. Any of them can be overwritten +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. -Each packages.yaml file begins with the string ``packages:`` and +Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` -applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, -or ``providers``. Each component has an ordered list of spec +applies settings to each package. Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. -Sometimes a package installation may have constraints that forbid +Sometimes a package installation may have constraints that forbid the first concretization rule, in which case Spack will use the first legal concretization rule. Going back to the example, if a user -requests gperftools 2.3 or later, then Spack will install version 2.4 +requests gperftools 2.3 or later, then Spack will install version 2.4 as the 2.4 version of gperftools is preferred over 2.3. -An explicit concretization rule in the preferred section will always -take preference over unlisted concretizations. In the above example, +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, xlc isn't listed in the compiler list. Every listed compiler from gcc to pgi will thus be preferred over the xlc compiler. @@ -2160,6 +2160,62 @@ package, this allows us to avoid race conditions in the library's build system. +.. _sanity-checks: + +Sanity checking an intallation +-------------------------------- + +By default, Spack assumes that a build has failed if nothing is +written to the install prefix, and that it has succeeded if anything +(a file, a directory, etc.) is written to the install prefix after +``install()`` completes. + +Consider a simple autotools build like this: + +.. code-block:: python + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +If you are using using standard autotools or CMake, ``configure`` and +``make`` will not write anything to the install prefix. Only ``make +install`` writes the files, and only once the build is already +complete. Not all builds are like this. Many builds of scientific +software modify the install prefix *before* ``make install``. Builds +like this can falsely report that they were successfully installed if +an error occurs before the install is complete but after files have +been written to the ``prefix``. + + +``sanity_check_is_file`` and ``sanity_check_is_dir`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can optionally specify *sanity checks* to deal with this problem. +Add properties like this to your package: + +.. code-block:: python + + class MyPackage(Package): + ... + + sanity_check_is_file = ['include/libelf.h'] + sanity_check_is_dir = [lib] + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +Now, after ``install()`` runs, Spack will check whether +``$prefix/include/libelf.h`` exists and is a file, and whether +``$prefix/lib`` exists and is a directory. If the checks fail, then +the build will fail and the install prefix will be removed. If they +succeed, Spack considers the build succeeful and keeps the prefix in +place. + + .. _file-manipulation: File manipulation functions diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 02fb3e58348..291056a7b93 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -318,16 +318,17 @@ class SomePackage(Package): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False - """List of prefix-relative file paths. If these do not exist after - install, or if they exist but are not files, sanity checks fail. + """List of prefix-relative file paths (or a single path). If these do + not exist after install, or if they exist but are not files, + sanity checks fail. """ - sanity_check_files = [] + sanity_check_is_file = [] - """List of prefix-relative directory paths. If these do not exist - after install, or if they exist but are not directories, sanity - checks will fail. + """List of prefix-relative directory paths (or a single path). If + these do not exist after install, or if they exist but are not + directories, sanity checks will fail. """ - sanity_check_dirs = [] + sanity_check_is_dir = [] def __init__(self, spec): @@ -966,14 +967,17 @@ def build_process(): def sanity_check_prefix(self): """This function checks whether install succeeded.""" def check_paths(path_list, filetype, predicate): + if isinstance(path_list, basestring): + path_list = [path_list] + for path in path_list: abs_path = os.path.join(self.prefix, path) if not predicate(abs_path): raise InstallError("Install failed for %s. No such %s in prefix: %s" % (self.name, filetype, path)) - check_paths(self.sanity_check_files, 'file', os.path.isfile) - check_paths(self.sanity_check_dirs, 'directory', os.path.isdir) + check_paths(self.sanity_check_is_file, 'file', os.path.isfile) + check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 0fcb56c164d..9f16708af5b 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,8 +38,7 @@ class Libelf(Package): provides('elf') - sanity_check_files = ['include/libelf.h'] - sanity_check_dirs = ['lib'] + sanity_check_is_file = 'include/libelf.h' def install(self, spec, prefix): configure("--prefix=" + prefix, From 802acb4d166a7ca3ff2db3c0a0c95a3ad8a6a81a Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:43:40 -0400 Subject: [PATCH 295/339] Update PAPI to 5.4.3 --- var/spack/repos/builtin/packages/papi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9a..53d69e28d90 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 1b279cd7ff29e09ef9afa1a328b964487b5b2030 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:55:45 -0400 Subject: [PATCH 296/339] Update OpenBLAS to 0.2.16 --- var/spack/repos/builtin/packages/openblas/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 3c909360a41..e16d3fe89c1 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -5,6 +5,7 @@ class Openblas(Package): homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') # virtual dependency From 4cf4bf3a0343fc93d6f0c14dd89956fa03b8f738 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 11:05:59 +0100 Subject: [PATCH 297/339] openssl : solved glitch to prevent spack to freeze when the network is unreachable --- .../repos/builtin/packages/openssl/package.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 70afaf4038a..22d538531a5 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,4 +1,4 @@ -import urllib +import urllib2 import llnl.util.tty as tty from spack import * @@ -37,17 +37,22 @@ def url_for_version(self, version): version_number = '.'.join([str(x) for x in version[:-1]]) older_url = older.format(version_number=version_number, version_full=version) latest_url = latest.format(version=version) - response = urllib.urlopen(latest.format(version=version)) - if response.getcode() == 404: - openssl_url = older_url - # Checks if we already warned the user for this particular version of OpenSSL. - # If not we display a warning message and mark this version - if not warnings_given_to_user.get(version, False): - tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') - tty.warn('Consider updating to the latest version of this package.') - tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) - warnings_given_to_user[version] = True - else: + try: + response = urllib2.urlopen(latest.format(version=version), timeout=5) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: + openssl_url = latest_url + except urllib2.URLError: + tty.warn('Cannot connect to network to retrieve OpenSSL version. Using default url.') + warnings_given_to_user[version] = True openssl_url = latest_url # Store the computed URL openssl_urls[version] = openssl_url From 3b24db92b5ea64814b1294fc6c9c982bbaa9b3c6 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Thu, 17 Mar 2016 13:43:40 -0400 Subject: [PATCH 298/339] Update PAPI to 5.4.3 --- var/spack/repos/builtin/packages/papi/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9a..53d69e28d90 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') From 076cc764a78c387d0e0cc38047a9df8497416f90 Mon Sep 17 00:00:00 2001 From: Ben Morgan Date: Fri, 18 Mar 2016 11:26:22 +0000 Subject: [PATCH 299/339] Add additional suffixes for GCC compiler --- lib/spack/spack/compilers/gcc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 495b638a3aa..64214db32d3 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -40,7 +40,8 @@ class Gcc(Compiler): fc_names = ['gfortran'] # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] + # Homebrew and Linuxes may build gcc with -X, -X.Y suffixes + suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d'] # Named wrapper links within spack.build_env_path link_paths = {'cc' : 'gcc/gcc', From ec8cc2b52839007f0825815c8cfdee8f9a8629a6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 14:40:53 +0100 Subject: [PATCH 300/339] PYTHONPATH : python patches methods for its extensions --- lib/spack/spack/build_environment.py | 4 ++- lib/spack/spack/cmd/uninstall.py | 12 +++---- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/modules.py | 12 +++++-- lib/spack/spack/package.py | 10 ------ .../repos/builtin/packages/py-nose/package.py | 6 ++-- .../repos/builtin/packages/python/package.py | 34 +++++++++++++++++-- 7 files changed, 53 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68477145fef..c51fa584778 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -282,9 +282,11 @@ def setup_package(pkg): for mod in modules: set_module_variables_for_package(pkg, mod) - # Allow dependencies to set up environment as well. + # Allow dependencies to modify the module for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + # Allow dependencies to set up environment as well + for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.setup_dependent_environment(env, pkg.spec) # TODO : implement validation #validate(env) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index d01aa2136b0..2fab8f6f2a1 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -79,7 +79,7 @@ def uninstall(parser, args): try: # should work if package is known to spack pkgs.append(s.package) - except spack.repository.UnknownPackageError, e: + except spack.repository.UnknownPackageError as e: # The package.py file has gone away -- but still # want to uninstall. spack.Package(s).do_uninstall(force=True) @@ -94,11 +94,11 @@ def num_installed_deps(pkg): for pkg in pkgs: try: pkg.do_uninstall(force=args.force) - except PackageStillNeededError, e: + except PackageStillNeededError as e: tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print - print "The following packages depend on it:" + print() + print("The following packages depend on it:") display_specs(e.dependents, long=True) - print - print "You can use spack uninstall -f to force this action." + print() + print("You can use spack uninstall -f to force this action.") sys.exit(1) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 39ee4e203d6..da8f1aa1bc0 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -150,7 +150,7 @@ def remove_install_directory(self, spec): if os.path.exists(path): try: shutil.rmtree(path) - except exceptions.OSError, e: + except exceptions.OSError as e: raise RemoveFailedError(spec, path, e) path = os.path.dirname(path) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d192bbe004e..7303844229c 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -119,13 +119,13 @@ def __init__(cls, name, bases, dict): module_types[cls.name] = cls def __init__(self, spec=None): + self.spec = spec + self.pkg = spec.package # Just stored for convenience + # category in the modules system # TODO: come up with smarter category names. self.category = "spack" - self.spec = spec - self.pkg = spec.package # Just stored for convenience - # short description default is just the package + version # packages can provide this optional attribute self.short_description = spec.format("$_ $@") @@ -161,6 +161,12 @@ def write(self): # Environment modifications guessed by inspecting the installation prefix env = inspect_path(self.spec.prefix) + + # Let the extendee modify their extensions before asking for package-specific modifications + for extendee in self.pkg.extendees: + extendee_spec = self.spec[extendee] + extendee_spec.package.modify_module(self.pkg, extendee_spec, self.spec) + # Package-specific environment modifications self.spec.package.setup_environment(env) # TODO : implement site-specific modifications and filters diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 185e3ad2eea..72c84ec624f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1261,16 +1261,6 @@ def rpath_args(self): """Get the rpath args as a string, with -Wl,-rpath, for each element.""" return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) - -class PythonExtension(Package): - def setup_dependent_environment(self, env, dependent_spec): - pass - - def setup_environment(self, env): - site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - env.prepend_path('PYTHONPATH', site_packages[0]) - def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index e817b8eb513..4fee99098ed 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -1,12 +1,12 @@ from spack import * -from spack.package import PythonExtension -class PyNose(PythonExtension): + +class PyNose(Package): """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" homepage = "https://pypi.python.org/pypi/nose" - url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" + url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d46e1068b6c..c445d26369e 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -1,11 +1,14 @@ +import functools +import glob +import inspect import os import re from contextlib import closing -from llnl.util.lang import match_predicate -from spack.util.environment import * -from spack import * import spack +from llnl.util.lang import match_predicate +from spack import * +from spack.util.environment import * class Python(Package): @@ -111,6 +114,31 @@ def modify_module(self, module, spec, ext_spec): else: module.python = Executable(join_path(spec.prefix.bin, 'python')) + # The code below patches the any python extension to have good defaults for `setup_dependent_environment` and + # `setup_environment` only if the extension didn't override any of these functions explicitly. + def _setup_env(self, env): + site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + env.prepend_path('PYTHONPATH', site_packages[0]) + + def _setup_denv(self, env, extension_spec): + pass + + pkg_cls = type(ext_spec.package) # Retrieve the type we may want to patch + if 'python' in pkg_cls.extendees: + # List of overrides we are interested in + interesting_overrides = ['setup_environment', 'setup_dependent_environment'] + overrides_found = [ + (name, defining_cls) for name, _, defining_cls, _, in inspect.classify_class_attrs(pkg_cls) + if + name in interesting_overrides and # The attribute has the right name + issubclass(defining_cls, Package) and defining_cls is not Package # and is an actual override + ] + if not overrides_found: + # If no override were found go on patching + pkg_cls.setup_environment = functools.wraps(Package.setup_environment)(_setup_env) + pkg_cls.setup_dependent_environment = functools.wraps(Package.setup_dependent_environment)(_setup_denv) + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) From 5f6f2d5f51fc3700dce13c6aa035943bbc42f6f7 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 14:42:24 +0100 Subject: [PATCH 301/339] Revert "openssl : solved glitch to prevent spack to freeze when the network is unreachable" This reverts commit 4cf4bf3a0343fc93d6f0c14dd89956fa03b8f738. --- .../repos/builtin/packages/openssl/package.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 22d538531a5..70afaf4038a 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,4 +1,4 @@ -import urllib2 +import urllib import llnl.util.tty as tty from spack import * @@ -37,22 +37,17 @@ def url_for_version(self, version): version_number = '.'.join([str(x) for x in version[:-1]]) older_url = older.format(version_number=version_number, version_full=version) latest_url = latest.format(version=version) - try: - response = urllib2.urlopen(latest.format(version=version), timeout=5) - if response.getcode() == 404: - openssl_url = older_url - # Checks if we already warned the user for this particular version of OpenSSL. - # If not we display a warning message and mark this version - if not warnings_given_to_user.get(version, False): - tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') - tty.warn('Consider updating to the latest version of this package.') - tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) - warnings_given_to_user[version] = True - else: - openssl_url = latest_url - except urllib2.URLError: - tty.warn('Cannot connect to network to retrieve OpenSSL version. Using default url.') - warnings_given_to_user[version] = True + response = urllib.urlopen(latest.format(version=version)) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: openssl_url = latest_url # Store the computed URL openssl_urls[version] = openssl_url From 67ca2c704b9c367b41ce29342a06427e156a30e9 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 15:18:26 +0100 Subject: [PATCH 302/339] modules : fixed bug in `modify_module` arguments --- lib/spack/spack/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7303844229c..f5bf2138456 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -165,7 +165,7 @@ def write(self): # Let the extendee modify their extensions before asking for package-specific modifications for extendee in self.pkg.extendees: extendee_spec = self.spec[extendee] - extendee_spec.package.modify_module(self.pkg, extendee_spec, self.spec) + extendee_spec.package.modify_module(self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications self.spec.package.setup_environment(env) From ccd90df62ffd69b5c3b4e1f8dbaf82146721a52a Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 15:41:14 +0100 Subject: [PATCH 303/339] modules : turned category into a property (logic can be extended later) --- lib/spack/spack/modules.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f5bf2138456..bb1a444e5a6 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -122,10 +122,6 @@ def __init__(self, spec=None): self.spec = spec self.pkg = spec.package # Just stored for convenience - # category in the modules system - # TODO: come up with smarter category names. - self.category = "spack" - # short description default is just the package + version # packages can provide this optional attribute self.short_description = spec.format("$_ $@") @@ -137,6 +133,17 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + @property + def category(self): + # Anything defined at the package level takes precedence + if hasattr(self.pkg, 'category'): + return self.pkg.category + # Extensions + for extendee in self.pkg.extendees: + return '{extendee} extension'.format(extendee=extendee) + # Not very descriptive fallback + return 'spack installed package' + # @property # def paths(self): # # Add python path unless it's an actual python installation From 1e468c55414822365b4ba7f7c52fbfdb5f30d3b2 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 16:02:44 +0100 Subject: [PATCH 304/339] modules : added formats mapping --- lib/spack/spack/modules.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index bb1a444e5a6..0378110c8cb 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -256,7 +256,10 @@ class TclModule(EnvModule): formats = { PrependPath: 'prepend-path {name} \"{value}\"\n', - SetEnv: 'setenv {name} \"{value}\"\n' + AppendPath: 'append-path {name} \"{value}\"\n', + RemovePath: 'remove-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' } @property From 491babd5cd435fc307b6e977b850e4e64d2b0ccf Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 18 Mar 2016 17:09:20 +0100 Subject: [PATCH 305/339] env modifications : added a validation rule --- lib/spack/spack/build_environment.py | 6 +++--- lib/spack/spack/environment.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index c51fa584778..59b234624cf 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -34,8 +34,9 @@ import sys import spack +import llnl.util.tty as tty from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, concatenate_paths +from spack.environment import EnvironmentModifications, concatenate_paths, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -288,8 +289,7 @@ def setup_package(pkg): # Allow dependencies to set up environment as well for dependency_spec in pkg.spec.traverse(root=False): dependency_spec.package.setup_dependent_environment(env, pkg.spec) - # TODO : implement validation - #validate(env) + validate(env, tty.warn) env.apply_modifications() diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 6d214595a3e..74aef57fe88 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -202,3 +202,33 @@ def concatenate_paths(paths): string """ return ':'.join(str(item) for item in paths) + + +def set_or_unset_not_first(variable, changes, errstream): + """ + Check if we are going to set or unset something after other modifications have already been requested + """ + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + if indexes: + good = '\t \t{context} at {filename}:{lineno}' + nogood = '\t--->\t{context} at {filename}:{lineno}' + errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + for ii, item in enumerate(changes): + print_format = nogood if ii in indexes else good + errstream(print_format.format(**item.args)) + + +def validate(env, errstream): + """ + Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for + everything that was found + + Current checks: + - set or unset variables after other changes on the same variable + + Args: + env: list of environment modifications + """ + modifications = env.group_by_name() + for variable, list_of_changes in sorted(modifications.items()): + set_or_unset_not_first(variable, list_of_changes, errstream) From 1fa38689d87480bb0d291af9b9cf0dbcf7557eb5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 09:35:56 -0700 Subject: [PATCH 306/339] Created flatten_dependencies function that dummy packages can use to create sane install environments. --- lib/spack/spack/directory_layout.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 242eb1afa04..9f174040627 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -33,6 +33,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp +from llnl.util.link_tree import * from spack.spec import Spec from spack.error import SpackError @@ -131,6 +132,23 @@ def remove_extension(self, spec, ext_spec): raise NotImplementedError() + def flatten_dependencies(self, spec, flat_dir): + """Make each dependency of spec present in dir via symlink.""" + for dep in spec.traverse(root=False): + name = dep.name + + dep_path = self.path_for_spec(dep) + dep_files = LinkTree(dep_path) + + os.mkdir(flat_dir+'/'+name) + + conflict = dep_files.find_conflict(flat_dir+'/'+name) + if conflict: + raise DependencyConflictError(conflict) + + dep_files.merge(flat_dir+'/'+name) + + def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) From 80495e50f9b3a5de6d0aadb054a770a685939e79 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 09:46:18 -0700 Subject: [PATCH 307/339] added error class for error that should never come up --- lib/spack/spack/directory_layout.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 9f174040627..dfd0dc42e4f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -498,3 +498,10 @@ def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not activated."% ( ext_spec.short_spec, spec.short_spec)) + +class DependencyConflictError(SpackError): + """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): + super(DependencyConflictError, self).__init__( + "%s conflicts with another file in the flattened directory." %( + conflict)) From 6acb830263e663b57bc8c077b348c897b0d1f612 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 18 Mar 2016 13:03:55 -0500 Subject: [PATCH 308/339] Add support for .tar files --- lib/spack/spack/url.py | 2 +- lib/spack/spack/util/compression.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index ad551a6dedb..f51f05cad7d 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -142,7 +142,7 @@ def split_url_extension(path): def downloaded_file_extension(path): """This returns the type of archive a URL refers to. This is - sometimes confusing becasue of URLs like: + sometimes confusing because of URLs like: (1) https://github.com/petdance/ack/tarball/1.93_02 diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index ea1f233bce8..5ae58674280 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -27,13 +27,12 @@ from itertools import product from spack.util.executable import which -# Supported archvie extensions. +# Supported archive extensions. PRE_EXTS = ["tar"] EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] -# Add EXTS last so that .tar.gz is matched *before* tar.gz -ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS - +# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz +ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS def allowed_archive(path): return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) From 76672a4e34e37aec0223e67d140fd19e9115b5a6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:28:44 -0700 Subject: [PATCH 309/339] Refactoring flat_install --- lib/spack/spack/__init__.py | 3 +++ lib/spack/spack/directory_layout.py | 24 ----------------------- lib/spack/spack/package.py | 30 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ab78ecef308..155e190597b 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -174,3 +174,6 @@ import spack.util.executable from spack.util.executable import * __all__ += spack.util.executable.__all__ + +from spack.package import flat_install, flatten_dependencies, DependencyConflictError +__all__ += ['flat_install', 'flatten_dependencies', 'DependencyConflictError'] diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index dfd0dc42e4f..e275031387c 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -132,23 +132,6 @@ def remove_extension(self, spec, ext_spec): raise NotImplementedError() - def flatten_dependencies(self, spec, flat_dir): - """Make each dependency of spec present in dir via symlink.""" - for dep in spec.traverse(root=False): - name = dep.name - - dep_path = self.path_for_spec(dep) - dep_files = LinkTree(dep_path) - - os.mkdir(flat_dir+'/'+name) - - conflict = dep_files.find_conflict(flat_dir+'/'+name) - if conflict: - raise DependencyConflictError(conflict) - - dep_files.merge(flat_dir+'/'+name) - - def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) @@ -498,10 +481,3 @@ def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not activated."% ( ext_spec.short_spec, spec.short_spec)) - -class DependencyConflictError(SpackError): - """Raised when the dependencies cannot be flattened as asked for.""" - def __init__(self, conflict): - super(DependencyConflictError, self).__init__( - "%s conflicts with another file in the flattened directory." %( - conflict)) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 972a0410b9a..1a962268cf4 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1211,6 +1211,28 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) +def flat_install(pkg, spec, prefix): + """Execute a dummy install and flatten dependencies""" + os.mkdir(prefix+'/libs') + flatten_dependencies(spec, prefix+'/libs') + +def flatten_dependencies(spec, flat_dir): + """Make each dependency of spec present in dir via symlink.""" + for dep in spec.traverse(root=False): + name = dep.name + + dep_path = spack.install_layout.path_for_spec(dep) + dep_files = LinkTree(dep_path) + + os.mkdir(flat_dir+'/'+name) + + conflict = dep_files.find_conflict(flat_dir+'/'+name) + if conflict: + raise DependencyConflictError(conflict) + + dep_files.merge(flat_dir+'/'+name) + + def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) @@ -1348,3 +1370,11 @@ def __init__(self, path): class ActivationError(ExtensionError): def __init__(self, msg, long_msg=None): super(ActivationError, self).__init__(msg, long_msg) + + +class DependencyConflictError(spack.error.SpackError): + """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): + super(DependencyConflictError, self).__init__( + "%s conflicts with another file in the flattened directory." %( + conflict)) From af7e3cadde02b2ab9b6de2c588a41663141928dd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:34:07 -0700 Subject: [PATCH 310/339] cleanup --- lib/spack/spack/directory_layout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index e275031387c..242eb1afa04 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -33,7 +33,6 @@ import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp -from llnl.util.link_tree import * from spack.spec import Spec from spack.error import SpackError From 151b04637ef01b8cde5967df6165bd245311b90a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 18 Mar 2016 11:55:31 -0700 Subject: [PATCH 311/339] changed function name and removed hardcoded libs dir --- lib/spack/spack/__init__.py | 4 ++-- lib/spack/spack/package.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 155e190597b..5d62d597cb7 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -175,5 +175,5 @@ from spack.util.executable import * __all__ += spack.util.executable.__all__ -from spack.package import flat_install, flatten_dependencies, DependencyConflictError -__all__ += ['flat_install', 'flatten_dependencies', 'DependencyConflictError'] +from spack.package import install_dependency_symlinks, flatten_dependencies, DependencyConflictError +__all__ += ['install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError'] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1a962268cf4..4c458522e05 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1211,10 +1211,9 @@ def rpath_args(self): return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) -def flat_install(pkg, spec, prefix): +def install_dependency_symlinks(pkg, spec, prefix): """Execute a dummy install and flatten dependencies""" - os.mkdir(prefix+'/libs') - flatten_dependencies(spec, prefix+'/libs') + flatten_dependencies(spec, prefix) def flatten_dependencies(spec, flat_dir): """Make each dependency of spec present in dir via symlink.""" From 23f3f1adfe01f7b1301959e7e54584724679c7b3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Mar 2016 13:32:37 -0700 Subject: [PATCH 312/339] Make InstallError and ExternalPackageError visiible in packages. --- lib/spack/spack/__init__.py | 8 ++++++-- lib/spack/spack/package.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 0ba42bbbfcd..aee11f061f3 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -189,5 +189,9 @@ from spack.util.executable import * __all__ += spack.util.executable.__all__ -from spack.package import install_dependency_symlinks, flatten_dependencies, DependencyConflictError -__all__ += ['install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError'] +from spack.package import \ + install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \ + InstallError, ExternalPackageError +__all__ += [ + 'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError', + 'InstallError', 'ExternalPackageError'] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b488e4c49db..dafad0b184a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1351,6 +1351,10 @@ def __init__(self, message, long_msg=None): super(InstallError, self).__init__(message, long_msg) +class ExternalPackageError(InstallError): + """Raised by install() when a package is only for external use.""" + + class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" def __init__(self, spec, dependents): From e9126baaabbfaa5e12641b3d44a5c5fd49e43ca3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Mar 2016 13:42:45 -0700 Subject: [PATCH 313/339] Add docs for InstallError. --- lib/spack/docs/packaging_guide.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index c1077e44977..519c0da2329 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1844,6 +1844,20 @@ dedicated process. .. _prefix-objects: + +Failing the build +---------------------- + +Sometimes you don't want a package to successfully install unless some +condition is true. You can explicitly cause the build to fail from +``install()`` by raising an ``InstallError``, for example: + +.. code-block:: python + + if spec.architecture.startswith('darwin'): + raise InstallError('This package does not build on Mac OS X!') + + Prefix objects ---------------------- From f5f7d47be1a5fd8488ed4cdc692173fe136d9791 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 19 Mar 2016 18:42:06 -0400 Subject: [PATCH 314/339] Support Bash "-u" mode --- share/spack/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 586a5b836b1..764af684001 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -141,7 +141,7 @@ function _spack_pathadd { fi # Do the actual prepending here. - eval "_pa_oldvalue=\$${_pa_varname}" + eval "_pa_oldvalue=\${${_pa_varname}:-}" if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then if [ -n "$_pa_oldvalue" ]; then From ed0b5e649dd6bc480e3faa6ffcca121eb9e264b8 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Fri, 18 Mar 2016 15:54:26 -0600 Subject: [PATCH 315/339] + Provide fpic variant for netlib-lapack. --- var/spack/repos/builtin/packages/netlib-lapack/package.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index 741f4af4215..78c5a053fe5 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -20,13 +20,13 @@ class NetlibLapack(Package): version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') variant('shared', default=False, description="Build shared library version") + variant('fpic', default=False, description="Build with -fpic compiler option") # virtual dependency provides('lapack') # blas is a virtual dependency. depends_on('blas') - depends_on('cmake') # Doesn't always build correctly in parallel @@ -37,24 +37,23 @@ def get_blas_libs(self): blas = self.spec['netlib-blas'] return [join_path(blas.prefix.lib, 'blas.a')] - @when('^atlas') def get_blas_libs(self): blas = self.spec['atlas'] return [join_path(blas.prefix.lib, l) for l in ('libf77blas.a', 'libatlas.a')] - def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] if '+shared' in spec: cmake_args.append('-DBUILD_SHARED_LIBS=ON') + if '+fpic' in spec: + cmake_args.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON') cmake_args += std_cmake_args cmake(*cmake_args) make() make("install") - From 620c120503c8c23d10feb14e14d37dfc36f4befb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 19 Mar 2016 19:32:44 +0100 Subject: [PATCH 316/339] fix openblas suffix for OS-X --- var/spack/repos/builtin/packages/openblas/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index e16d3fe89c1..781a1e2ec8f 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,4 +1,5 @@ from spack import * +import sys class Openblas(Package): """OpenBLAS: An optimized BLAS library""" @@ -16,13 +17,14 @@ def install(self, spec, prefix): make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') make('install', "PREFIX='%s'" % prefix) + lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' # Blas virtual package should provide blas.a and libblas.a with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') - symlink('libopenblas.so', 'libblas.so') + symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') - symlink('libopenblas.so', 'liblapack.so') + symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix) From 95ad2875b6bb4dc15eb43b44c1dbdd521c843119 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sun, 20 Mar 2016 09:25:02 +0100 Subject: [PATCH 317/339] fix scalapack suffix for osx --- .../builtin/packages/netlib-scalapack/package.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 22d538560e8..a5b6eedafbe 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -1,8 +1,9 @@ from spack import * +import sys class NetlibScalapack(Package): """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" - + homepage = "http://www.netlib.org/scalapack/" url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" @@ -32,17 +33,17 @@ def install(self, spec, prefix): "-DCMAKE_C_FLAGS=-fPIC", "-DCMAKE_Fortran_FLAGS=-fPIC" ]) - + options.extend(std_cmake_args) - + with working_dir('spack-build', create=True): cmake('..', *options) make() make("install") def setup_dependent_environment(self, module, spec, dependent_spec): - # TODO treat OS that are not Linux... - lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a' + lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' + lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib spec['scalapack'].cc_link = spec['scalapack'].fc_link From db61a09cf14cc791d7692aac0329e8958b228fb6 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 18 Mar 2016 16:28:36 +0100 Subject: [PATCH 318/339] fix mumps: when installed with parmetis has to be linked with metis --- .../repos/builtin/packages/mumps/package.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 44a37903cc8..5c120c37df8 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -20,10 +20,10 @@ class Mumps(Package): variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') - + depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') - depends_on('metis', when='~parmetis+metis') + depends_on('metis', when='+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') depends_on('lapack') @@ -38,11 +38,11 @@ class Mumps(Package): def write_makefile_inc(self): if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') - + makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] orderings = ['-Dpord'] - + if '+ptscotch' in self.spec or '+scotch' in self.spec: join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') makefile_conf.extend( @@ -54,15 +54,19 @@ def write_makefile_inc(self): if '+ptscotch' in self.spec: orderings.append('-Dptscotch') - if '+parmetis' in self.spec or '+metis' in self.spec: + if '+parmetis' in self.spec and '+metis' in self.spec: libname = 'parmetis' if '+parmetis' in self.spec else 'metis' makefile_conf.extend( - ["IMETIS = -I%s" % self.spec[libname].prefix.include, - "LMETIS = -L%s -l%s" % (self.spec[libname].prefix.lib, libname)]) + ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + + orderings.append('-Dparmetis') + elif '+metis' in self.spec: + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) orderings.append('-Dmetis') - if '+parmetis' in self.spec: - orderings.append('-Dparmetis') makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) @@ -101,12 +105,12 @@ def write_makefile_inc(self): # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER makefile_conf.append("CDEFS = -DAdd_") - + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), 'Makefile.inc') with open(makefile_inc_template, "r") as fh: makefile_conf.extend(fh.read().split('\n')) - + with working_dir('.'): with open("Makefile.inc", "w") as fh: makefile_inc = '\n'.join(makefile_conf) @@ -130,7 +134,7 @@ def install(self, spec, prefix): make_libs.append('zexamples') self.write_makefile_inc() - + make(*make_libs) install_tree('lib', prefix.lib) From 7b283bfaffa5509915d91392878203bba8a8d2cb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 07:48:17 +0100 Subject: [PATCH 319/339] fix petsc on osx --- .../repos/builtin/packages/petsc/package.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index efe172fc083..7239baaf7f5 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -61,14 +61,24 @@ def mpi_dependent_options(self): errors = ['incompatible variants given'] + errors raise RuntimeError('\n'.join(errors)) else: - compiler_opts = [ - '--with-mpi=1', - '--with-mpi-dir=%s' % self.spec['mpi'].prefix, - ] + if self.compiler.name == "clang": + compiler_opts = [ + '--with-mpi=1', + '--with-cc=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # Avoid confusing PETSc config by clang: warning: argument unused during compilation + '--with-cxx=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + '--with-fc=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + '--with-f77=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif77'), + ] + else: + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] return compiler_opts def install(self, spec, prefix): - options = [] + options = ['--with-debugging=0', + '--with-ssl=0'] options.extend(self.mpi_dependent_options()) options.extend([ '--with-precision=%s' % ('double' if '+double' in spec else 'single'), From aef6d50b08daf893174046ba14c09a7019fe1212 Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Mar 2016 09:23:25 +0100 Subject: [PATCH 320/339] uninstall : fixed typo (print statement vs. print function) --- lib/spack/spack/cmd/uninstall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 2fab8f6f2a1..1ece8386124 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -96,9 +96,9 @@ def num_installed_deps(pkg): pkg.do_uninstall(force=args.force) except PackageStillNeededError as e: tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print() + print('') print("The following packages depend on it:") display_specs(e.dependents, long=True) - print() + print('') print("You can use spack uninstall -f to force this action.") sys.exit(1) From b926d8a0cdcf47de029e372cf15e281266930378 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 14:33:00 +0100 Subject: [PATCH 321/339] build parmetis against external metis --- .../parmetis/enable_external_metis.patch | 13 ++++++++++ .../builtin/packages/parmetis/package.py | 24 ++++--------------- 2 files changed, 17 insertions(+), 20 deletions(-) create mode 100644 var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch diff --git a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch new file mode 100644 index 00000000000..514781b8b88 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ca945dd..1bf94e9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -33,7 +33,7 @@ include_directories(${GKLIB_PATH}) + include_directories(${METIS_PATH}/include) + + # List of directories that cmake will look for CMakeLists.txt +-add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) ++#add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) + add_subdirectory(include) + add_subdirectory(libparmetis) + add_subdirectory(programs) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index c897dec7e44..3f18dd76b87 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -25,9 +25,6 @@ from spack import * -# FIXME : lot of code is duplicated from packages/metis/package.py . Inheriting from there may reduce -# FIXME : the installation rules to just a few lines - class Parmetis(Package): """ @@ -43,13 +40,11 @@ class Parmetis(Package): variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') - variant('idx64', default=False, description='Use int64_t as default index type') - variant('double', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:') # build dependency depends_on('mpi') - # FIXME : this should conflict with metis as it builds its own version internally + patch('enable_external_metis.patch') + depends_on('metis') depends_on('gdb', when='+gdb') @@ -63,8 +58,8 @@ def install(self, spec, prefix): # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case # FIXME : they use a non-standard name - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), - '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=metis_source), + options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), # still need headers from METIS source, and they are not installed with METIS. shame... + '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), '-DCMAKE_C_COMPILER:STRING=mpicc', '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) @@ -78,18 +73,7 @@ def install(self, spec, prefix): if '+gdb' in spec: options.append('-DGDB:BOOL=ON') - metis_header = join_path(metis_source, 'include', 'metis.h') - - if '+idx64' in spec: - filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) - - if '+double' in spec: - filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) - with working_dir(build_directory, create=True): cmake(source_directory, *options) make() make("install") - # Parmetis build system doesn't allow for an external metis to be used, but doesn't copy the required - # metis header either - install(metis_header, self.prefix.include) From 3fcaf5b9031c3d55b152844b97cd432078340f4b Mon Sep 17 00:00:00 2001 From: alalazo Date: Mon, 21 Mar 2016 14:47:05 +0100 Subject: [PATCH 322/339] llvm : update to 3.8.0 --- var/spack/repos/builtin/packages/llvm/package.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 280e400f696..5c38b5b88c3 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -132,6 +132,21 @@ class Llvm(Package): 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', } }, + { + 'version' : '3.8.0', + 'md5':'07a7a74f3c6bd65de4702bf941b511a0', + 'resources' : { + 'compiler-rt' : 'd6fcbe14352ffb708e4d1ac2e48bb025', + 'openmp' : '8fd7cc35d48051613cf1e750e9f22e40', + 'polly' : '1b3b20f52d34a4024e21a4ea7112caa7', + 'libcxx' : 'd6e0bdbbee39f7907ad74fd56d03b88a', + 'libcxxabi' : 'bbe6b4d72c7c5978550d370af529bcf7', + 'clang' : 'cc99e7019bb74e6459e80863606250c5', + 'clang-tools-extra' : 'c2344f50e0eea0b402f0092a80ddc036', + 'lldb' : 'a5da35ed9cc8c8817ee854e3dbfba00e', + 'llvm-libunwind' : '162ade468607f153cca12be90b5194fa', + } + }, { 'version' : '3.7.1', 'md5':'bf8b3a2c79e61212c5409041dfdbd319', From 3a3443dff63167fc0331672871af4c0cc1fe5903 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 15:14:48 +0100 Subject: [PATCH 323/339] parmetis patches by PETSc developers --- .../builtin/packages/parmetis/package.py | 6 ++ ...9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch | 77 +++++++++++++++++++ ...9d68aa1d6cbc70740d0f35024aae17f7d5cb.patch | 35 +++++++++ 3 files changed, 118 insertions(+) create mode 100644 var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch create mode 100644 var/spack/repos/builtin/packages/parmetis/pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 3f18dd76b87..c691cf4191d 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -46,6 +46,12 @@ class Parmetis(Package): patch('enable_external_metis.patch') depends_on('metis') + # bug fixes from PETSc developers + # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ + patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') + # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ + patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') + depends_on('gdb', when='+gdb') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch new file mode 100644 index 00000000000..e6b8056c210 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch @@ -0,0 +1,77 @@ +From 1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b Mon Sep 17 00:00:00 2001 +From: Jed Brown +Date: Fri, 12 Oct 2012 15:45:10 -0500 +Subject: [PATCH] ParMetis bug fixes reported by John Fettig [petsc-maint + #133631] + +''' +I have also reported to to Karypis but have received zero +response and he hasn't released any updates to the original release +either. At least he approved my forum posting so that other people +can see the bug and the fix. +http://glaros.dtc.umn.edu/gkhome/node/837 +''' + +Hg-commit: 1c2b9fe39201d404b493885093b5992028b9b8d4 +--- + libparmetis/xyzpart.c | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 3a2c289..63abfcb 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -104,7 +104,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + for (i=0; i +Date: Tue, 20 Mar 2012 11:59:44 -0500 +Subject: [PATCH] parmetis: fix bug reported by jfettig; '<' to '<=' in xyzpart + +Hg-commit: 2dd2eae596acaabbc80e0ef875182616f868dbc2 +--- + libparmetis/xyzpart.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 307aed9..3a2c289 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -111,7 +111,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + /* determine bucket counts */ + iset(nbins, 0, lcounts); + for (j=0, i=0; i Date: Mon, 21 Mar 2016 16:18:02 +0100 Subject: [PATCH 324/339] llvm : does not support python 3 --- var/spack/repos/builtin/packages/llvm/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index 5c38b5b88c3..1d25d59e504 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -52,7 +52,7 @@ class Llvm(Package): depends_on('cmake @2.8.12.2:') # Universal dependency - depends_on('python@2.7:') + depends_on('python@2.7:2.8') # Seems not to support python 3.X.Y # lldb dependencies depends_on('ncurses', when='+lldb') From c5c92d50eb78673c635a7ed26145bf10a15c20cb Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:24:36 -0400 Subject: [PATCH 325/339] paraview: remove trailing whitespace --- var/spack/repos/builtin/packages/paraview/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index ccf2d14c06e..b3312736f16 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -37,11 +37,11 @@ class Paraview(Package): #depends_on('protobuf') # version mismatches? #depends_on('sqlite') # external version not supported depends_on('zlib') - + def url_for_version(self, version): """Handle ParaView version-based custom URLs.""" return self._url_str % (version.up_to(2), version) - + def install(self, spec, prefix): with working_dir('spack-build', create=True): From 8e77f1776020c2a6edf9eb284615d99c905821e9 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:54:58 -0400 Subject: [PATCH 326/339] paraview: fix variant description typo --- var/spack/repos/builtin/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index b3312736f16..0c3c7c16214 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -16,7 +16,7 @@ class Paraview(Package): variant('osmesa', default=False, description='Enable OSMesa support') variant('qt', default=False, description='Enable Qt support') - variant('opengl2', default=False, description='Enable OPengl2 backend') + variant('opengl2', default=False, description='Enable OpenGL2 backend') depends_on('python', when='+python') depends_on('py-numpy', when='+python') From 553fff270a13d40fb9a41569bebf5075a740d8ac Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 21 Mar 2016 11:24:49 -0400 Subject: [PATCH 327/339] paraview: disallow python3 ParaView is not Python3-ready. --- var/spack/repos/builtin/packages/paraview/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index 0c3c7c16214..c16054816c1 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -18,7 +18,7 @@ class Paraview(Package): variant('qt', default=False, description='Enable Qt support') variant('opengl2', default=False, description='Enable OpenGL2 backend') - depends_on('python', when='+python') + depends_on('python@2:2.7', when='+python') depends_on('py-numpy', when='+python') depends_on('py-matplotlib', when='+python') depends_on('tcl', when='+tcl') From 953fb35f82c099936f833f31dcd292ffa1a3ec06 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 21 Mar 2016 18:00:32 +0100 Subject: [PATCH 328/339] add superlu_dist package --- .../builtin/packages/superlu-dist/package.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 var/spack/repos/builtin/packages/superlu-dist/package.py diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py new file mode 100644 index 00000000000..c4c76909b35 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -0,0 +1,63 @@ +from spack import * + +class SuperluDist(Package): + """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" + + version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae') + version('4.2', 'ae9fafae161f775fbac6eba11e530a65') + version('4.1', '4edee38cc29f687bd0c8eb361096a455') + version('4.0', 'c0b98b611df227ae050bc1635c6940e0') + + depends_on ('mpi') + depends_on ('blas') + depends_on ('lapack') + depends_on ('parmetis') + depends_on ('metis') + + def install(self, spec, prefix): + makefile_inc = [] + makefile_inc.extend([ + 'PLAT = _mac_x', + 'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix + 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', + 'BLASDEF = -DUSE_VENDOR_BLAS', + 'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names + 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, + 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, + 'FLIBS =', + 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', + 'ARCH = ar', + 'ARCHFLAGS = cr', + 'RANLIB = true', + 'CC = mpicc', # FIXME avoid hardcoding MPI compiler names + 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), + 'NOOPTS = -fPIC -std=c99', + 'FORTRAN = mpif77', + 'F90FLAGS = -O2', + 'LOADER = mpif77', + 'LOADOPTS =', + 'CDEFS = -DAdd_' + ]) + + #with working_dir('src'): + with open('make.inc', 'w') as fh: + fh.write('\n'.join(makefile_inc)) + + make("lib", parallel=False) + + # FIXME: + # cd "EXAMPLE" do + # system "make" + + # need to install by hand + headers_location = join_path(self.prefix.include,'superlu_dist') + mkdirp(headers_location) + # FIXME: fetch all headers in the folder automatically + for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']: + superludist_header = join_path(self.stage.source_path, 'SRC/',header) + install(superludist_header, headers_location) + + superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') + install(superludist_lib,self.prefix.lib) From b79fce76cb4d412cf718b48b05d549470e4c55ab Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 21 Mar 2016 14:28:07 -0700 Subject: [PATCH 329/339] Fix issue 573 where Spack was ignoring user's compiler preference in concretization --- lib/spack/spack/concretize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8083f919828..2e576743ec6 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -241,7 +241,7 @@ def concretize_compiler(self, spec): return False #Find the another spec that has a compiler, or the root if none do - other_spec = find_spec(spec, lambda(x) : x.compiler) + other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -288,7 +288,7 @@ def find_spec(spec, condition): if condition(spec): return spec - return None # Nohting matched the condition. + return None # Nothing matched the condition. def cmp_specs(lhs, rhs): From 5d06daeb5eed28b4b91ba62a8f99165d87b5ef86 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 21 Mar 2016 14:28:34 -0700 Subject: [PATCH 330/339] Add test for issue 573, child with compiler not respected in concretization --- lib/spack/spack/test/concretize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f264faf17a2..08cce096747 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -309,3 +309,10 @@ def test_find_spec_none(self): Spec('d')), Spec('e')) self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) + + + def test_compiler_child(self): + s = Spec('mpileaks%clang ^dyninst%gcc') + s.concretize() + self.assertTrue(s['mpileaks'].satisfies('%clang')) + self.assertTrue(s['dyninst'].satisfies('%gcc')) From 48b35bb495bc154c64d23d1ad364506c4572b913 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 00:11:18 -0700 Subject: [PATCH 331/339] Fix print function in uninstall.py --- lib/spack/spack/cmd/uninstall.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 1ece8386124..350ef372cb6 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from __future__ import print_function import sys import argparse @@ -63,12 +64,12 @@ def uninstall(parser, args): matching_specs = spack.installed_db.query(spec) if not args.all and len(matching_specs) > 1: tty.error("%s matches multiple packages:" % spec) - print + print() display_specs(matching_specs, long=True) - print - print "You can either:" - print " a) Use a more specific spec, or" - print " b) use spack uninstall -a to uninstall ALL matching specs." + print() + print("You can either:") + print(" a) Use a more specific spec, or") + print(" b) use spack uninstall -a to uninstall ALL matching specs.") sys.exit(1) if len(matching_specs) == 0: From e88df95b42fdcaa49552811853f8ca4ecc52cf9f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 00:35:13 -0700 Subject: [PATCH 332/339] Remove unused code from modules.py --- lib/spack/spack/modules.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 0378110c8cb..4e98d500013 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -133,6 +133,7 @@ def __init__(self, spec=None): if self.spec.package.__doc__: self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) + @property def category(self): # Anything defined at the package level takes precedence @@ -144,21 +145,6 @@ def category(self): # Not very descriptive fallback return 'spack installed package' - # @property - # def paths(self): - # # Add python path unless it's an actual python installation - # # TODO : is there a better way to do this? - # # FIXME : add PYTHONPATH to every python package - # if self.spec.name != 'python': - # site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - # if site_packages: - # add_path('PYTHONPATH', site_packages[0]) - # - # # FIXME : Same for GEM_PATH - # if self.spec.package.extends(spack.spec.Spec('ruby')): - # add_path('GEM_PATH', self.spec.prefix) - # - # return self._paths def write(self): """Write out a module file for this object.""" @@ -166,16 +152,20 @@ def write(self): if not os.path.exists(module_dir): mkdirp(module_dir) - # Environment modifications guessed by inspecting the installation prefix + # Environment modifications guessed by inspecting the + # installation prefix env = inspect_path(self.spec.prefix) - # Let the extendee modify their extensions before asking for package-specific modifications + # Let the extendee modify their extensions before asking for + # package-specific modifications for extendee in self.pkg.extendees: extendee_spec = self.spec[extendee] - extendee_spec.package.modify_module(self.pkg.module, extendee_spec, self.spec) + extendee_spec.package.modify_module( + self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications self.spec.package.setup_environment(env) + # TODO : implement site-specific modifications and filters if not env: return @@ -232,7 +222,7 @@ def file_name(self): def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, self.spec.compiler.name, - self.spec.compiler.version, + self.spec.compiler.version, self.spec.dag_hash()) def write_header(self, dk_file): @@ -270,7 +260,7 @@ def file_name(self): def use_name(self): return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, self.spec.compiler.name, - self.spec.compiler.version, + self.spec.compiler.version, self.spec.dag_hash()) def write_header(self, module_file): From 439d47b4e45c674ab9aa4ebd0c2bfaf6911ade60 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 01:48:18 -0700 Subject: [PATCH 333/339] Refactor environment setup. - Gave setup_environment and setup_dependent_environment more similar signatures. They now allows editing the Spack env and the runtime env for *this* package and dependents, respectively. - modify_module renamed to setup_dependent_python_module for symmetry with setup_dependent_environment and to avoid confusion with environment modules. - removed need for patching Package objects at runtime. - adjust packages to reflect these changes. --- lib/spack/spack/build_environment.py | 35 +++-- lib/spack/spack/modules.py | 3 +- lib/spack/spack/package.py | 120 ++++++++++++++---- .../repos/builtin/packages/mpich/package.py | 9 +- .../packages/netlib-scalapack/package.py | 2 +- .../repos/builtin/packages/openmpi/package.py | 16 +-- .../repos/builtin/packages/python/package.py | 39 ++---- .../repos/builtin/packages/qt/package.py | 10 +- .../repos/builtin/packages/ruby/package.py | 12 +- 9 files changed, 150 insertions(+), 96 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 59b234624cf..5688d47e2d3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -84,7 +84,7 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) -def set_compiler_environment_variables(pkg): +def set_compiler_environment_variables(pkg, env): assert pkg.spec.concrete # Set compiler variables used by CMake and autotools assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) @@ -92,7 +92,6 @@ def set_compiler_environment_variables(pkg): # Populate an object with the list of environment modifications # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. - env = EnvironmentModifications() link_dir = spack.build_env_path env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) @@ -113,7 +112,7 @@ def set_compiler_environment_variables(pkg): return env -def set_build_environment_variables(pkg): +def set_build_environment_variables(pkg, env): """ This ensures a clean install environment when we build packages """ @@ -134,7 +133,6 @@ def set_build_environment_variables(pkg): if os.path.isdir(ci): env_paths.append(ci) - env = EnvironmentModifications() for item in reversed(env_paths): env.prepend_path('PATH', item) env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) @@ -180,7 +178,7 @@ def set_build_environment_variables(pkg): return env -def set_module_variables_for_package(pkg, m): +def set_module_variables_for_package(pkg, module): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ @@ -190,6 +188,8 @@ def set_module_variables_for_package(pkg, m): jobs = 1 elif pkg.make_jobs: jobs = pkg.make_jobs + + m = module m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. @@ -271,9 +271,12 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - env = EnvironmentModifications() - env.extend(set_compiler_environment_variables(pkg)) - env.extend(set_build_environment_variables(pkg)) + spack_env = EnvironmentModifications() + run_env = EnvironmentModifications() + + set_compiler_environment_variables(pkg, spack_env) + set_build_environment_variables(pkg, spack_env) + # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from # an existing class like spack.repos.original.libelf.Libelf, @@ -285,12 +288,20 @@ def setup_package(pkg): # Allow dependencies to modify the module for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.modify_module(pkg.module, dependency_spec, pkg.spec) + dpkg = dependency_spec.package + dpkg.setup_dependent_python_module(pkg.module, pkg.spec) + # Allow dependencies to set up environment as well for dependency_spec in pkg.spec.traverse(root=False): - dependency_spec.package.setup_dependent_environment(env, pkg.spec) - validate(env, tty.warn) - env.apply_modifications() + dpkg = dependency_spec.package + dpkg.setup_dependent_environment(spack_env, run_env, pkg.spec) + + # Allow the package to apply some settings. + pkg.setup_environment(spack_env, run_env) + + # Make sure nothing's strange about the Spack environment. + validate(spack_env, tty.warn) + spack_env.apply_modifications() def fork(pkg, function): diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 4e98d500013..05c93cd3e65 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -164,7 +164,8 @@ def write(self): self.pkg.module, extendee_spec, self.spec) # Package-specific environment modifications - self.spec.package.setup_environment(env) + spack_env = EnvironmentModifications() + self.spec.package.setup_environment(spack_env, env) # TODO : implement site-specific modifications and filters if not env: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index acad5a28f6c..9d8ac87bd70 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1002,56 +1002,120 @@ def module(self): return __import__(self.__class__.__module__, fromlist=[self.__class__.__name__]) - def setup_environment(self, env): - """ - Appends in `env` the list of environment modifications needed to use this package outside of spack. + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environemnts for a package. - Default implementation does nothing, but this can be overridden if the package needs a particular environment. + `spack_env` and `run_env` are `EnvironmentModifications` + objects. Package authors can call methods on them to alter + the environment within Spack and at runtime. - Example : + Both `spack_env` and `run_env` are applied within the build + process, before this package's `install()` method is called. - 1. A lot of Qt extensions need `QTDIR` set. This can be used to do that. + Modifications in `run_env` will *also* be added to the + generated environment modules for this package. + + Default implementation does nothing, but this can be + overridden if the package needs a particular environment. + + Examples: + + 1. Qt extensions need `QTDIR` set. Args: - env: list of environment modifications to be updated + spack_env (EnvironmentModifications): list of + modifications to be applied when this package is built + within Spack. + + run_env (EnvironmentModifications): list of environment + changes to be applied when this package is run outside + of Spack. + """ pass - def setup_dependent_environment(self, env, dependent_spec): - """ - Called before the install() method of dependents. - Appends in `env` the list of environment modifications needed by dependents (or extensions) during the - installation of a package. The default implementation delegates to `setup_environment`, but can be overridden - if the modifications to the environment happen to be different from the one needed to use the package outside - of spack. + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one. - This is useful if there are some common steps to installing all extensions for a certain package. + This is similar to `setup_environment`, but it is used to + modify the compile and runtime environments of packages that + *depend* on this one. This gives packages like Python and + others that follow the extension model a way to implement + common environment or compile-time settings for dependencies. + + By default, this delegates to self.setup_environment() Example : - 1. Installing python modules generally requires `PYTHONPATH` to point to the lib/pythonX.Y/site-packages - directory in the module's install prefix. This could set that variable. + 1. Installing python modules generally requires + `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could + set that variable. Args: - env: list of environment modifications to be updated - dependent_spec: dependent (or extension) of this spec - """ - self.setup_environment(env) - def modify_module(self, module, spec, dependent_spec): + spack_env (EnvironmentModifications): list of + modifications to be applied when the dependent package + is bulit within Spack. + + run_env (EnvironmentModifications): list of environment + changes to be applied when the dependent package is + run outside of Spack. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + """ + self.setup_environment(spack_env, run_env) + + + def setup_dependent_python_module(self, module, dependent_spec): + """Set up Python module-scope variables for dependent packages. + Called before the install() method of dependents. - Default implementation does nothing, but this can be overridden by an extendable package to set up the module of - its extensions. This is useful if there are some common steps to installing all extensions for a - certain package. + Default implementation does nothing, but this can be + overridden by an extendable package to set up the module of + its extensions. This is useful if there are some common steps + to installing all extensions for a certain package. Example : - 1. Extensions often need to invoke the 'python' interpreter from the Python installation being extended. - This routine can put a 'python' Executable object in the module scope for the extension package to simplify - extension installs. + 1. Extensions often need to invoke the `python` + interpreter from the Python installation being + extended. This routine can put a 'python' Executable + object in the module scope for the extension package to + simplify extension installs. + + 2. MPI compilers could set some variables in the + dependent's scope that point to `mpicc`, `mpicxx`, + etc., allowing them to be called by common names + regardless of which MPI is used. + + 3. BLAS/LAPACK implementations can set some variables + indicating the path to their libraries, since these + paths differ by BLAS/LAPACK implementation. + + Args: + + module (module): The Python `module` object of the + dependent package. Packages can use this to set + module-scope variables for the dependent to use. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + """ pass diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 5af9b585ea8..90b5d42eab4 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -47,13 +47,6 @@ class Mpich(Package): provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def setup_environment(self, env): - env.set_env('MPICH_CC', self.compiler.cc) - env.set_env('MPICH_CXX', self.compiler.cxx) - env.set_env('MPICH_F77', self.compiler.f77) - env.set_env('MPICH_F90', self.compiler.fc) - env.set_env('MPICH_FC', self.compiler.fc) - def setup_dependent_environment(self, env, dependent_spec): env.set_env('MPICH_CC', spack_cc) env.set_env('MPICH_CXX', spack_cxx) @@ -61,7 +54,7 @@ def setup_dependent_environment(self, env, dependent_spec): env.set_env('MPICH_F90', spack_f90) env.set_env('MPICH_FC', spack_fc) - def modify_module(self, module, spec, dep_spec): + def setup_dependent_python_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 36f506f7cdf..62abfcc48e3 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -40,7 +40,7 @@ def install(self, spec, prefix): make() make("install") - def modify_module(self, module, spec, dependent_spec): + def setup_dependent_python_module(self, module, spec, dependent_spec): lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 7783ca87666..c91a13e3760 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -41,17 +41,13 @@ class Openmpi(Package): def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_environment(self, env): - env.set_env('OMPI_CC', self.compiler.cc) - env.set_env('OMPI_CXX', self.compiler.cxx) - env.set_env('OMPI_FC', self.compiler.fc) - env.set_env('OMPI_F77', self.compiler.f77) - def setup_dependent_environment(self, env, dependent_spec): - env.set_env('OMPI_CC', spack_cc) - env.set_env('OMPI_CXX', spack_cxx) - env.set_env('OMPI_FC', spack_fc) - env.set_env('OMPI_F77', spack_f77) + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set_env('OMPI_CC', spack_cc) + spack_env.set_env('OMPI_CXX', spack_cxx) + spack_env.set_env('OMPI_FC', spack_fc) + spack_env.set_env('OMPI_F77', spack_f77) + def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index c445d26369e..593a27708c8 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -92,13 +92,21 @@ def python_include_dir(self): def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - def setup_dependent_environment(self, env, extension_spec): - # Set PYTHONPATH to include site-packages dir for the extension and any other python extensions it depends on. + + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. python_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - env.set_env('PYTHONPATH', ':'.join(python_paths)) + + pythonpath = ':'.join(python_paths) + spack_env.set_env('PYTHONPATH', pythonpath) + run_env.set_env('PYTHONPATH', pythonpath) + def modify_module(self, module, spec, ext_spec): """ @@ -114,31 +122,6 @@ def modify_module(self, module, spec, ext_spec): else: module.python = Executable(join_path(spec.prefix.bin, 'python')) - # The code below patches the any python extension to have good defaults for `setup_dependent_environment` and - # `setup_environment` only if the extension didn't override any of these functions explicitly. - def _setup_env(self, env): - site_packages = glob.glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - env.prepend_path('PYTHONPATH', site_packages[0]) - - def _setup_denv(self, env, extension_spec): - pass - - pkg_cls = type(ext_spec.package) # Retrieve the type we may want to patch - if 'python' in pkg_cls.extendees: - # List of overrides we are interested in - interesting_overrides = ['setup_environment', 'setup_dependent_environment'] - overrides_found = [ - (name, defining_cls) for name, _, defining_cls, _, in inspect.classify_class_attrs(pkg_cls) - if - name in interesting_overrides and # The attribute has the right name - issubclass(defining_cls, Package) and defining_cls is not Package # and is an actual override - ] - if not overrides_found: - # If no override were found go on patching - pkg_cls.setup_environment = functools.wraps(Package.setup_environment)(_setup_env) - pkg_cls.setup_dependent_environment = functools.wraps(Package.setup_dependent_environment)(_setup_denv) - # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 35b9d684624..039aeb3c314 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -55,8 +55,14 @@ class Qt(Package): depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def setup_environment(self, env): - env.set_env['QTDIR'] = self.prefix + + def setup_environment(self, spack_env, env): + env.set_env('QTDIR', self.prefix) + + + def setup_dependent_environment(self, spack_env, run_env, dspec): + spack_env.set_env('QTDIR', self.prefix) + def patch(self): if self.spec.satisfies('@4'): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 2d1da8c9af1..39f65f51d2c 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -2,7 +2,7 @@ class Ruby(Package): - """A dynamic, open source programming language with a focus on + """A dynamic, open source programming language with a focus on simplicity and productivity.""" homepage = "https://www.ruby-lang.org/" @@ -17,15 +17,17 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_environment(self, env, extension_spec): + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. # Set GEM_PATH to include dependent gem directories ruby_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): ruby_paths.append(d.prefix) - env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + + spack_env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) # The actual installation path for this gem - env.set_env('GEM_HOME', extension_spec.prefix) + spack_env.set_env('GEM_HOME', extension_spec.prefix) def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME @@ -38,5 +40,3 @@ def modify_module(self, module, spec, ext_spec): # Ruby extension builds have global ruby and gem functions module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - - From b1516f64eb75c108eded1e9ee7e0480a4552236a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Mar 2016 02:21:31 -0700 Subject: [PATCH 334/339] Rename some environment methods to be less repetitive, add set_path. --- lib/spack/spack/build_environment.py | 45 ++++++++++--------- lib/spack/spack/environment.py | 22 ++++++++- lib/spack/spack/test/environment.py | 22 +++++---- .../repos/builtin/packages/mpich/package.py | 10 ++--- .../repos/builtin/packages/openmpi/package.py | 8 ++-- .../repos/builtin/packages/python/package.py | 4 +- .../repos/builtin/packages/qt/package.py | 4 +- .../repos/builtin/packages/ruby/package.py | 5 ++- 8 files changed, 73 insertions(+), 47 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5688d47e2d3..fc5b7d6207f 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -36,7 +36,7 @@ import spack import llnl.util.tty as tty from llnl.util.filesystem import * -from spack.environment import EnvironmentModifications, concatenate_paths, validate +from spack.environment import EnvironmentModifications, validate from spack.util.environment import * from spack.util.executable import Executable, which @@ -93,22 +93,23 @@ def set_compiler_environment_variables(pkg, env): # and return it # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. link_dir = spack.build_env_path - env.set_env('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) - env.set_env('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) - env.set_env('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) - env.set_env('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) + env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) + env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) + env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) + # Set SPACK compiler variables so that our wrapper knows what to call compiler = pkg.compiler if compiler.cc: - env.set_env('SPACK_CC', compiler.cc) + env.set('SPACK_CC', compiler.cc) if compiler.cxx: - env.set_env('SPACK_CXX', compiler.cxx) + env.set('SPACK_CXX', compiler.cxx) if compiler.f77: - env.set_env('SPACK_F77', compiler.f77) + env.set('SPACK_F77', compiler.f77) if compiler.fc: - env.set_env('SPACK_FC', compiler.fc) + env.set('SPACK_FC', compiler.fc) - env.set_env('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) return env @@ -135,25 +136,25 @@ def set_build_environment_variables(pkg, env): for item in reversed(env_paths): env.prepend_path('PATH', item) - env.set_env(SPACK_ENV_PATH, concatenate_paths(env_paths)) + env.set_path(SPACK_ENV_PATH, env_paths) # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] - env.set_env(SPACK_DEPENDENCIES, concatenate_paths(dep_prefixes)) - env.set_env('CMAKE_PREFIX_PATH', concatenate_paths(dep_prefixes)) # Add dependencies to CMAKE_PREFIX_PATH + env.set_path(SPACK_DEPENDENCIES, dep_prefixes) + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH # Install prefix - env.set_env(SPACK_PREFIX, pkg.prefix) + env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - env.set_env(SPACK_INSTALL, spack.install_path) + env.set(SPACK_INSTALL, spack.install_path) # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - env.unset_env('LD_LIBRARY_PATH') - env.unset_env('LD_RUN_PATH') - env.unset_env('DYLD_LIBRARY_PATH') + env.unset('LD_LIBRARY_PATH') + env.unset('LD_RUN_PATH') + env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) @@ -162,9 +163,9 @@ def set_build_environment_variables(pkg, env): # Working directory for the spack command itself, for debug logs. if spack.debug: - env.set_env(SPACK_DEBUG, 'TRUE') - env.set_env(SPACK_SHORT_SPEC, pkg.spec.short_spec) - env.set_env(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) + env.set(SPACK_DEBUG, 'TRUE') + env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) + env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] @@ -173,7 +174,7 @@ def set_build_environment_variables(pkg, env): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - env.set_env('PKG_CONFIG_PATH', concatenate_paths(pkg_config_dirs)) + env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) return env diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 74aef57fe88..72aafa4e2d6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -29,6 +29,12 @@ def execute(self): os.environ.pop(self.name, None) # Avoid throwing if the variable was not set +class SetPath(NameValueModifier): + def execute(self): + string_path = concatenate_paths(self.value) + os.environ[self.name] = string_path + + class AppendPath(NameValueModifier): def execute(self): environment_value = os.environ.get(self.name, '') @@ -103,7 +109,7 @@ def _get_outside_caller_attributes(self): } return args - def set_env(self, name, value, **kwargs): + def set(self, name, value, **kwargs): """ Stores in the current object a request to set an environment variable @@ -115,7 +121,7 @@ def set_env(self, name, value, **kwargs): item = SetEnv(name, value, **kwargs) self.env_modifications.append(item) - def unset_env(self, name, **kwargs): + def unset(self, name, **kwargs): """ Stores in the current object a request to unset an environment variable @@ -126,6 +132,18 @@ def unset_env(self, name, **kwargs): item = UnsetEnv(name, **kwargs) self.env_modifications.append(item) + def set_path(self, name, elts, **kwargs): + """ + Stores a request to set a path generated from a list. + + Args: + name: name o the environment variable to be set. + elts: elements of the path to set. + """ + kwargs.update(self._get_outside_caller_attributes()) + item = SetPath(name, elts, **kwargs) + self.env_modifications.append(item) + def append_path(self, name, path, **kwargs): """ Stores in the current object a request to append a path to a path list diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 3e03760c016..6c8f5ea43c3 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -11,21 +11,27 @@ def setUp(self): os.environ['PATH_LIST'] = '/path/second:/path/third' os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' - def test_set_env(self): + def test_set(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value') - env.set_env('B', 3) + env.set('A', 'dummy value') + env.set('B', 3) env.apply_modifications() self.assertEqual('dummy value', os.environ['A']) self.assertEqual(str(3), os.environ['B']) - def test_unset_env(self): + def test_unset(self): env = EnvironmentModifications() self.assertEqual('foo', os.environ['UNSET_ME']) - env.unset_env('UNSET_ME') + env.unset('UNSET_ME') env.apply_modifications() self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') + def test_set_path(self): + env = EnvironmentModifications() + env.set_path('A', ['foo', 'bar', 'baz']) + env.apply_modifications() + self.assertEqual('foo:bar:baz', os.environ['A']) + def test_path_manipulation(self): env = EnvironmentModifications() @@ -51,7 +57,7 @@ def test_path_manipulation(self): def test_extra_arguments(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value', who='Pkg1') + env.set('A', 'dummy value', who='Pkg1') for x in env: assert 'who' in x.args env.apply_modifications() @@ -59,8 +65,8 @@ def test_extra_arguments(self): def test_extend(self): env = EnvironmentModifications() - env.set_env('A', 'dummy value') - env.set_env('B', 3) + env.set('A', 'dummy value') + env.set('B', 3) copy_construct = EnvironmentModifications(env) self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 90b5d42eab4..c4d9940bb72 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -48,11 +48,11 @@ class Mpich(Package): provides('mpi@:1.3', when='@1:') def setup_dependent_environment(self, env, dependent_spec): - env.set_env('MPICH_CC', spack_cc) - env.set_env('MPICH_CXX', spack_cxx) - env.set_env('MPICH_F77', spack_f77) - env.set_env('MPICH_F90', spack_f90) - env.set_env('MPICH_FC', spack_fc) + env.set('MPICH_CC', spack_cc) + env.set('MPICH_CXX', spack_cxx) + env.set('MPICH_F77', spack_f77) + env.set('MPICH_F90', spack_f90) + env.set('MPICH_FC', spack_fc) def setup_dependent_python_module(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index c91a13e3760..9a127f18123 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -43,10 +43,10 @@ def url_for_version(self, version): def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - spack_env.set_env('OMPI_CC', spack_cc) - spack_env.set_env('OMPI_CXX', spack_cxx) - spack_env.set_env('OMPI_FC', spack_fc) - spack_env.set_env('OMPI_F77', spack_f77) + spack_env.set('OMPI_CC', spack_cc) + spack_env.set('OMPI_CXX', spack_cxx) + spack_env.set('OMPI_FC', spack_fc) + spack_env.set('OMPI_F77', spack_f77) def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 593a27708c8..4f55bc803ea 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -104,8 +104,8 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) pythonpath = ':'.join(python_paths) - spack_env.set_env('PYTHONPATH', pythonpath) - run_env.set_env('PYTHONPATH', pythonpath) + spack_env.set('PYTHONPATH', pythonpath) + run_env.set('PYTHONPATH', pythonpath) def modify_module(self, module, spec, ext_spec): diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index 039aeb3c314..d08e8e81e1a 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -57,11 +57,11 @@ class Qt(Package): def setup_environment(self, spack_env, env): - env.set_env('QTDIR', self.prefix) + env.set('QTDIR', self.prefix) def setup_dependent_environment(self, spack_env, run_env, dspec): - spack_env.set_env('QTDIR', self.prefix) + spack_env.set('QTDIR', self.prefix) def patch(self): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 39f65f51d2c..7ff1898ce91 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -25,9 +25,10 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): if d.package.extends(self.spec): ruby_paths.append(d.prefix) - spack_env.set_env('GEM_PATH', concatenate_paths(ruby_paths)) + spack_env.set_path('GEM_PATH', ruby_paths) + # The actual installation path for this gem - spack_env.set_env('GEM_HOME', extension_spec.prefix) + spack_env.set('GEM_HOME', extension_spec.prefix) def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME From 23bf70296e4a02f49c301b1b744055802b1e8dc1 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:22:42 +0100 Subject: [PATCH 335/339] arpack: add a patch for 3.3 --- .../repos/builtin/packages/arpack-ng/package.py | 4 ++++ .../builtin/packages/arpack-ng/pdlamch10.patch | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 0b49d142024..614071cf534 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -35,6 +35,10 @@ class ArpackNg(Package): variant('shared', default=True, description='Enables the build of shared libraries') variant('mpi', default=False, description='Activates MPI support') + # The function pdlamch10 does not set the return variable. This is fixed upstream + # see https://github.com/opencollab/arpack-ng/issues/34 + patch('pdlamch10.patch', when='@3.3:') + depends_on('blas') depends_on('lapack') depends_on('mpi', when='+mpi') diff --git a/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch new file mode 100644 index 00000000000..922828909f8 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch @@ -0,0 +1,15 @@ +diff --git a/PARPACK/SRC/MPI/pdlamch10.f b/PARPACK/SRC/MPI/pdlamch10.f +index 6571da9..2882c2e 100644 +--- a/PARPACK/SRC/MPI/pdlamch10.f ++++ b/PARPACK/SRC/MPI/pdlamch10.f +@@ -86,8 +86,8 @@ + TEMP = TEMP1 + END IF + * +- PDLAMCH = TEMP ++ PDLAMCH10 = TEMP + * +-* End of PDLAMCH ++* End of PDLAMCH10 + * + END From aca8b5c89de7058ab58f2eb3392e8b023b16fe8a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:27:00 +0100 Subject: [PATCH 336/339] move suite sparse to a proper folder --- .../builtin/packages/{SuiteSparse => suite-sparse}/package.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename var/spack/repos/builtin/packages/{SuiteSparse => suite-sparse}/package.py (100%) diff --git a/var/spack/repos/builtin/packages/SuiteSparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py similarity index 100% rename from var/spack/repos/builtin/packages/SuiteSparse/package.py rename to var/spack/repos/builtin/packages/suite-sparse/package.py From 3c5a1605d5e3cba881b743bf561f0966b1a3ad8b Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:27:39 +0100 Subject: [PATCH 337/339] adjust the class name for suite sparse --- var/spack/repos/builtin/packages/suite-sparse/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py index 6e130d118fe..b57f9967c30 100644 --- a/var/spack/repos/builtin/packages/suite-sparse/package.py +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -1,7 +1,7 @@ from spack import * -class Suitesparse(Package): +class SuiteSparse(Package): """ SuiteSparse is a suite of sparse matrix algorithms """ @@ -24,4 +24,3 @@ def install(self, spec, prefix): # FIXME : this actually uses the current workaround # FIXME : (blas / lapack always provide libblas and liblapack as aliases) make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') - From cb97e8dd9301e5bbc6752caed2f4861a2a06f494 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Tue, 22 Mar 2016 09:43:10 +0100 Subject: [PATCH 338/339] adjust eigen and octave to use new names of suite sparse --- var/spack/repos/builtin/packages/eigen/package.py | 2 +- var/spack/repos/builtin/packages/octave/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index e40046b4529..8d6e672f86c 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -48,7 +48,7 @@ class Eigen(Package): depends_on('metis', when='+metis') depends_on('scotch', when='+scotch') depends_on('fftw', when='+fftw') - depends_on('SuiteSparse', when='+suitesparse') + depends_on('suite-sparse', when='+suitesparse') depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 depends_on('gmp') diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py index 38b355159dd..6e99c236520 100644 --- a/var/spack/repos/builtin/packages/octave/package.py +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -62,7 +62,7 @@ class Octave(Package): depends_on('qrupdate', when='+qrupdate') #depends_on('qscintilla', when='+qscintilla) # TODO: add package depends_on('qt', when='+qt') - depends_on('SuiteSparse', when='+suitesparse') + depends_on('suite-sparse',when='+suitesparse') depends_on('zlib', when='+zlib') From a26992ef55fed958c15b45b989fc0a4d57f02251 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Mar 2016 01:56:16 -0700 Subject: [PATCH 339/339] Change from PR #552: rename setup_dependent_python_module -> setup_dependent_package - Fixed in package.py - Fixed wrong prototypes in packages that use it. - Fixed build_environment to set module variables properly - added hacky fix to ensure spec/package consistency in build processes. - Need to think about defensive spec copy done by `Repo.get`. May be time to think about an immutable spec implementation. --- lib/spack/spack/build_environment.py | 52 ++++++++++++++----- lib/spack/spack/package.py | 2 +- .../repos/builtin/packages/mpich/package.py | 2 +- .../packages/netlib-scalapack/package.py | 22 ++++---- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fc5b7d6207f..119a255a349 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -225,7 +225,7 @@ def set_module_variables_for_package(pkg, module): m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) - m.spack_f90 = join_path(link_dir, pkg.compiler.link_paths['fc']) + m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) # Emulate some shell commands for convenience m.pwd = os.getcwd @@ -270,32 +270,56 @@ def parent_class_modules(cls): return result +def setup_module_variables_for_dag(pkg): + """Set module-scope variables for all packages in the DAG.""" + for spec in pkg.spec.traverse(order='post'): + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + spkg = spec.package + modules = parent_class_modules(spkg.__class__) + for mod in modules: + set_module_variables_for_package(spkg, mod) + set_module_variables_for_package(spkg, spkg.module) + + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() run_env = EnvironmentModifications() + # Before proceeding, ensure that specs and packages are consistent + # + # This is a confusing behavior due to how packages are + # constructed. `setup_dependent_package` may set attributes on + # specs in the DAG for use by other packages' install + # method. However, spec.package will look up a package via + # spack.repo, which defensively copies specs into packages. This + # code ensures that all packages in the DAG have pieces of the + # same spec object at build time. + # + # This is safe for the build process, b/c the build process is a + # throwaway environment, but it is kind of dirty. + # + # TODO: Think about how to avoid this fix and do something cleaner. + for s in pkg.spec.traverse(): s.package.spec = s + set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - - # If a user makes their own package repo, e.g. - # spack.repos.mystuff.libelf.Libelf, and they inherit from - # an existing class like spack.repos.original.libelf.Libelf, - # then set the module variables for both classes so the - # parent class can still use them if it gets called. - modules = parent_class_modules(pkg.__class__) - for mod in modules: - set_module_variables_for_package(pkg, mod) + setup_module_variables_for_dag(pkg) # Allow dependencies to modify the module - for dependency_spec in pkg.spec.traverse(root=False): + spec = pkg.spec + for dependency_spec in spec.traverse(root=False): dpkg = dependency_spec.package - dpkg.setup_dependent_python_module(pkg.module, pkg.spec) + dpkg.setup_dependent_package(pkg.module, spec) # Allow dependencies to set up environment as well - for dependency_spec in pkg.spec.traverse(root=False): + for dependency_spec in spec.traverse(root=False): dpkg = dependency_spec.package - dpkg.setup_dependent_environment(spack_env, run_env, pkg.spec) + dpkg.setup_dependent_environment(spack_env, run_env, spec) # Allow the package to apply some settings. pkg.setup_environment(spack_env, run_env) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9d8ac87bd70..9af32218374 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1075,7 +1075,7 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec): self.setup_environment(spack_env, run_env) - def setup_dependent_python_module(self, module, dependent_spec): + def setup_dependent_package(self, module, dependent_spec): """Set up Python module-scope variables for dependent packages. Called before the install() method of dependents. diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index c4d9940bb72..b20dc8dd60c 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -54,7 +54,7 @@ def setup_dependent_environment(self, env, dependent_spec): env.set('MPICH_F90', spack_f90) env.set('MPICH_FC', spack_fc) - def setup_dependent_python_module(self, module, spec, dep_spec): + def setup_dependent_package(self, module, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? module.mpicc = join_path(self.prefix.bin, 'mpicc') diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index 62abfcc48e3..c3e6822cdfe 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -1,4 +1,5 @@ from spack import * +import sys class NetlibScalapack(Package): """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" @@ -11,16 +12,16 @@ class NetlibScalapack(Package): version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') # versions before 2.0.0 are not using cmake and requires blacs as # a separated package - + variant('shared', default=True, description='Build the shared library version') variant('fpic', default=False, description="Build with -fpic compiler option") - + provides('scalapack') - + depends_on('mpi') depends_on('lapack') - - def install(self, spec, prefix): + + def install(self, spec, prefix): options = [ "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), @@ -40,10 +41,11 @@ def install(self, spec, prefix): make() make("install") - def setup_dependent_python_module(self, module, spec, dependent_spec): + def setup_dependent_package(self, module, dependent_spec): + spec = self.spec lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' - lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a' + lib_suffix = lib_dsuffix if '+shared' in spec else '.a' - spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib - spec['scalapack'].cc_link = spec['scalapack'].fc_link - spec['scalapack'].libraries = [join_path(spec['scalapack'].prefix.lib, 'libscalapack%s' % lib_suffix)] + spec.fc_link = '-L%s -lscalapack' % spec.prefix.lib + spec.cc_link = spec.fc_link + spec.libraries = [join_path(spec.prefix.lib, 'libscalapack%s' % lib_suffix)]