From 6f69c01915aad8bb2d4b242ab7e5b185adf08b7d Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 27 Apr 2016 15:00:47 -0400 Subject: [PATCH 01/30] bootstrap: fall back to the default upstream URL Fixes #352. --- lib/spack/spack/cmd/bootstrap.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index bec11439b5c..9fd428e6b1e 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -31,6 +31,8 @@ import spack from spack.util.executable import which +_SPACK_UPSTREAM = 'https://github.com/llnl/spack' + description = "Create a new installation of spack in another prefix" def setup_parser(subparser): @@ -40,9 +42,15 @@ def setup_parser(subparser): def get_origin_url(): git_dir = join_path(spack.prefix, '.git') git = which('git', required=True) - origin_url = git( - '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - output=str) + try: + origin_url = git( + '--git-dir=%s' % git_dir, + 'config', '--get', 'remote.origin.url', + output=str) + except ProcessError: + origin_url = _SPACK_UPSTREAM + tty.warn('No git repository found; ' + 'using default upstream URL: %s' % origin_url) return origin_url.strip() From 7ec191ce0b82827013485a98db84cd66aa2ca1b4 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 31 May 2016 10:44:01 -0400 Subject: [PATCH 02/30] bootstrap: use the currently checked out branch The `master` branch is not really where Spack development happens, so default to it, but use the user's current branch if it's there. --- lib/spack/spack/cmd/bootstrap.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 9fd428e6b1e..16808b30b88 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -39,9 +39,14 @@ def setup_parser(subparser): subparser.add_argument('prefix', help="names of prefix where we should install spack") -def get_origin_url(): +def get_origin_info(): git_dir = join_path(spack.prefix, '.git') git = which('git', required=True) + try: + branch = git('symbolic-ref', '--short', 'HEAD', output=str) + except ProcessError: + branch = 'develop' + tty.warn('No branch found; using default branch: %s' % branch) try: origin_url = git( '--git-dir=%s' % git_dir, @@ -51,11 +56,11 @@ def get_origin_url(): origin_url = _SPACK_UPSTREAM tty.warn('No git repository found; ' 'using default upstream URL: %s' % origin_url) - return origin_url.strip() + return (origin_url.strip(), branch.strip()) def bootstrap(parser, args): - origin_url = get_origin_url() + origin_url, branch = get_origin_info() prefix = args.prefix tty.msg("Fetching spack from origin: %s" % origin_url) @@ -81,8 +86,9 @@ def bootstrap(parser, args): git = which('git', required=True) git('init', '--shared', '-q') git('remote', 'add', 'origin', origin_url) - git('fetch', 'origin', 'master:refs/remotes/origin/master', '-n', '-q') - git('reset', '--hard', 'origin/master', '-q') + git('fetch', 'origin', '%s:refs/remotes/origin/%s' % (branch, branch), + '-n', '-q') + git('reset', '--hard', 'origin/%s' % branch, '-q') tty.msg("Successfully created a new spack in %s" % prefix, "Run %s/bin/spack to use this installation." % prefix) From 89bf5f4045eebdc1f40561dbb70a56e170411c20 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 31 May 2016 10:49:24 -0400 Subject: [PATCH 03/30] bootstrap: allow using alternate remotes If you want to bootstrap from a fork, the `--remote` option may be used to select it. Also limit the branches to 'develop' and 'master' if the remote is 'origin' since those are the actual integration branches used (other branches on 'origin' are just PR branches). --- lib/spack/spack/cmd/bootstrap.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 16808b30b88..3480c96f39a 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -36,10 +36,13 @@ description = "Create a new installation of spack in another prefix" def setup_parser(subparser): + subparser.add_argument( + '-r', '--remote', action='store', dest='remote', + help="name of the remote to bootstrap from", default='origin') subparser.add_argument('prefix', help="names of prefix where we should install spack") -def get_origin_info(): +def get_origin_info(remote): git_dir = join_path(spack.prefix, '.git') git = which('git', required=True) try: @@ -47,10 +50,14 @@ def get_origin_info(): except ProcessError: branch = 'develop' tty.warn('No branch found; using default branch: %s' % branch) + if remote == 'origin' and \ + branch not in ('master', 'develop'): + branch = 'develop' + tty.warn('Unknown branch found; using default branch: %s' % branch) try: origin_url = git( '--git-dir=%s' % git_dir, - 'config', '--get', 'remote.origin.url', + 'config', '--get', 'remote.%s.url' % remote, output=str) except ProcessError: origin_url = _SPACK_UPSTREAM @@ -60,10 +67,10 @@ def get_origin_info(): def bootstrap(parser, args): - origin_url, branch = get_origin_info() + origin_url, branch = get_origin_info(args.remote) prefix = args.prefix - tty.msg("Fetching spack from origin: %s" % origin_url) + tty.msg("Fetching spack from '%s': %s" % (args.remote, origin_url)) if os.path.isfile(prefix): tty.die("There is already a file at %s" % prefix) From e3e94f0ac96db3c1ddd51555fec5a2bf71b867bf Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 31 May 2016 10:55:14 -0400 Subject: [PATCH 04/30] bootstrap: name the current branch the same as the remote --- lib/spack/spack/cmd/bootstrap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 3480c96f39a..04892e258db 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -96,6 +96,7 @@ def bootstrap(parser, args): git('fetch', 'origin', '%s:refs/remotes/origin/%s' % (branch, branch), '-n', '-q') git('reset', '--hard', 'origin/%s' % branch, '-q') + git('checkout', '-B', branch, 'origin/%s' % branch, '-q') tty.msg("Successfully created a new spack in %s" % prefix, "Run %s/bin/spack to use this installation." % prefix) From 5f720f9b7c0e507c2a0335ce9840036ad0b9262f Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 9 Jun 2016 10:51:29 -0400 Subject: [PATCH 05/30] flake8: appease the style checker --- lib/spack/spack/cmd/bootstrap.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 04892e258db..60e2bd3a114 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from subprocess import check_call import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp @@ -35,11 +34,14 @@ description = "Create a new installation of spack in another prefix" + def setup_parser(subparser): subparser.add_argument( '-r', '--remote', action='store', dest='remote', help="name of the remote to bootstrap from", default='origin') - subparser.add_argument('prefix', help="names of prefix where we should install spack") + subparser.add_argument( + 'prefix', + help="names of prefix where we should install spack") def get_origin_info(remote): @@ -82,7 +84,8 @@ def bootstrap(parser, args): files_in_the_way = os.listdir(prefix) if files_in_the_way: - tty.die("There are already files there! Delete these files before boostrapping spack.", + tty.die("There are already files there! " + "Delete these files before boostrapping spack.", *files_in_the_way) tty.msg("Installing:", From 7551e4c14f2f88c00f147684b01a72a69e3ecce5 Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Sun, 10 Jul 2016 18:02:51 -0700 Subject: [PATCH 06/30] cmake@3.6.0 Add CMake version 3.6.0. --- var/spack/repos/builtin/packages/cmake/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index 7b2a125fe50..d7c8f646a0a 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -30,6 +30,7 @@ class Cmake(Package): homepage = 'https://www.cmake.org' url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' + version('3.6.0', 'aa40fbecf49d99c083415c2411d12db9') version('3.5.2', '701386a1b5ec95f8d1075ecf96383e02') version('3.5.1', 'ca051f4a66375c89d1a524e726da0296') version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') From b7d9b58cc5de8a36d43cc082cac0e1ccb6e05f74 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Tue, 12 Jul 2016 19:54:44 +0200 Subject: [PATCH 07/30] Fix preferred providers. --- lib/spack/spack/preferred_packages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 48205841501..e873fa3c7ee 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -41,7 +41,7 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): pkglist.append('all') for pkg in pkglist: order = self.preferred.get(pkg, {}).get(component, {}) - if type(order) is dict: + if isinstance(order, dict) and second_key: order = order.get(second_key, {}) if not order: continue @@ -89,9 +89,9 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, # a and b are considered to match entries in the sorting list if they # satisfy the list component. def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): - if not a or not a.concrete: + if not a or (not a.concrete and not second_key): return -1 - if not b or not b.concrete: + if not b or (not b.concrete and not second_key): return 1 specs = self._spec_for_pkgname(pkgname, component, second_key) a_index = None From 52ca215e2947f7fe5c5248ccbd5e4fa19c8536fd Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 13 Jul 2016 10:50:57 +0200 Subject: [PATCH 08/30] mkl: add provide blas/lapack --- .../repos/builtin/packages/mkl/package.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py index 454e78d29ca..889851c9530 100644 --- a/var/spack/repos/builtin/packages/mkl/package.py +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -18,6 +18,11 @@ class Mkl(IntelInstaller): version('11.3.3.210', 'f72546df27f5ebb0941b5d21fd804e34', url="file://%s/l_mkl_11.3.3.210.tgz" % os.getcwd()) + # virtual dependency + provides('blas') + provides('lapack') + # TODO: MKL also provides implementation of Scalapack. + def install(self, spec, prefix): self.intel_prefix = os.path.join(prefix, "pkg") @@ -26,3 +31,21 @@ def install(self, spec, prefix): mkl_dir = os.path.join(self.intel_prefix, "mkl") for f in os.listdir(mkl_dir): os.symlink(os.path.join(mkl_dir, f), os.path.join(self.prefix, f)) + + def setup_dependent_package(self, module, dspec): + # For now use Single Dynamic Library: + # To set the threading layer at run time, use the + # mkl_set_threading_layer function or set MKL_THREADING_LAYER + # variable to one of the following values: INTEL, SEQUENTIAL, PGI. + # To set interface layer at run time, use the mkl_set_interface_layer + # function or set the MKL_INTERFACE_LAYER variable to LP64 or ILP64. + + # Otherwise one would need to specify several libraries + # (e.g. mkl_intel_lp64;mkl_sequential;mkl_core), which reflect + # different interface and threading layers. + + name = 'libmkl_rt.%s' % dso_suffix + libdir = find_library_path(name, self.prefix.lib64, self.prefix.lib) + + self.spec.blas_shared_lib = join_path(libdir, name) + self.spec.lapack_shared_lib = self.spec.blas_shared_lib From 5ffc50732bdcfca325115e23d359612426f38854 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Wed, 13 Jul 2016 23:05:08 +0200 Subject: [PATCH 09/30] flake8 fixes --- lib/spack/spack/preferred_packages.py | 99 +++++++++++++++------------ 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index e873fa3c7ee..1b94f03de7f 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -26,8 +26,10 @@ import spack from spack.version import * + class PreferredPackages(object): - _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent + # Arbitrary, but consistent + _default_order = {'compiler': ['gcc', 'intel', 'clang', 'pgi', 'xlc']} def __init__(self): self.preferred = spack.config.get_config('packages') @@ -35,7 +37,8 @@ def __init__(self): # Given a package name, sort component (e.g, version, compiler, ...), and # a second_key (used by providers), return the list - def _order_for_package(self, pkgname, component, second_key, test_all=True): + def _order_for_package(self, pkgname, component, second_key, + test_all=True): pkglist = [pkgname] if test_all: pkglist.append('all') @@ -48,11 +51,11 @@ def _order_for_package(self, pkgname, component, second_key, test_all=True): return [str(s).strip() for s in order] return [] - # A generic sorting function. Given a package name and sort # component, return less-than-0, 0, or greater-than-0 if # a is respectively less-than, equal to, or greater than b. - def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + def _component_compare(self, pkgname, component, a, b, + reverse_natural_compare, second_key): if a is None: return -1 if b is None: @@ -84,11 +87,11 @@ def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, else: return 0 - # A sorting function for specs. Similar to component_compare, but # a and b are considered to match entries in the sorting list if they # satisfy the list component. - def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + def _spec_compare(self, pkgname, component, a, b, + reverse_natural_compare, second_key): if not a or (not a.concrete and not second_key): return -1 if not b or (not b.concrete and not second_key): @@ -98,78 +101,88 @@ def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, secon b_index = None reverse = -1 if reverse_natural_compare else 1 for i, cspec in enumerate(specs): - if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)): + if a_index is None and (cspec.satisfies(a) or a.satisfies(cspec)): a_index = i if b_index: break - if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)): + if b_index is None and (cspec.satisfies(b) or b.satisfies(cspec)): b_index = i if a_index: break - if a_index != None and b_index == None: return -1 - elif a_index == None and b_index != None: return 1 - elif a_index != None and b_index == a_index: return -1 * cmp(a, b) - elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) - else: return cmp(a, b) * reverse - - + if a_index is not None and b_index is None: + return -1 + elif a_index is None and b_index is not None: + return 1 + elif a_index is not None and b_index == a_index: + return -1 * cmp(a, b) + elif (a_index is not None and b_index is not None and + a_index != b_index): + return cmp(a_index, b_index) + else: + return cmp(a, b) * reverse # Given a sort order specified by the pkgname/component/second_key, return # a list of CompilerSpecs, VersionLists, or Specs for that sorting list. def _spec_for_pkgname(self, pkgname, component, second_key): key = (pkgname, component, second_key) - if not key in self._spec_for_pkgname_cache: + if key not in self._spec_for_pkgname_cache: pkglist = self._order_for_package(pkgname, component, second_key) if not pkglist: if component in self._default_order: pkglist = self._default_order[component] if component == 'compiler': - self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist] + self._spec_for_pkgname_cache[key] = \ + [spack.spec.CompilerSpec(s) for s in pkglist] elif component == 'version': - self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist] + self._spec_for_pkgname_cache[key] = \ + [VersionList(s) for s in pkglist] else: - self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] + self._spec_for_pkgname_cache[key] = \ + [spack.spec.Spec(s) for s in pkglist] return self._spec_for_pkgname_cache[key] - def provider_compare(self, pkgname, provider_str, a, b): - """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or - greater-than b. A and b are possible implementations of provider_str. - One provider is less-than another if it is preferred over the other. - For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if - mvapich should be preferred over openmpi for scorep.""" - return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) - + """Return less-than-0, 0, or greater than 0 if a is respecively + less-than, equal-to, or greater-than b. A and b are possible + implementations of provider_str. One provider is less-than another + if it is preferred over the other. For example, + provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would + return -1 if mvapich should be preferred over openmpi for scorep.""" + return self._spec_compare(pkgname, 'providers', a, b, False, + provider_str) def spec_has_preferred_provider(self, pkgname, provider_str): - """Return True iff the named package has a list of preferred provider""" - return bool(self._order_for_package(pkgname, 'providers', provider_str, False)) - + """Return True iff the named package has a list of preferred + providers""" + return bool(self._order_for_package(pkgname, 'providers', + provider_str, False)) def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is - respecively less-than, equal-to, or greater-than version b of pkgname. - One version is less-than another if it is preferred over the other.""" + respectively less-than, equal-to, or greater-than version b of + pkgname. One version is less-than another if it is preferred over + the other.""" return self._spec_compare(pkgname, 'version', a, b, True, None) - def variant_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if variant a of pkgname is - respecively less-than, equal-to, or greater-than variant b of pkgname. - One variant is less-than another if it is preferred over the other.""" + respectively less-than, equal-to, or greater-than variant b of + pkgname. One variant is less-than another if it is preferred over + the other.""" return self._component_compare(pkgname, 'variant', a, b, False, None) - def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. - One architecture is less-than another if it is preferred over the other.""" - return self._component_compare(pkgname, 'architecture', a, b, False, None) - + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname + is respectively less-than, equal-to, or greater-than architecture b + of pkgname. One architecture is less-than another if it is preferred + over the other.""" + return self._component_compare(pkgname, 'architecture', a, b, + False, None) def compiler_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is - respecively less-than, equal-to, or greater-than compiler b of pkgname. - One compiler is less-than another if it is preferred over the other.""" + respecively less-than, equal-to, or greater-than compiler b of + pkgname. One compiler is less-than another if it is preferred over + the other.""" return self._spec_compare(pkgname, 'compiler', a, b, False, None) From 668b4f1b2ce8e61457309d67b798e58ef481a08e Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Wed, 13 Jul 2016 23:36:23 +0200 Subject: [PATCH 10/30] mkl: set MKLROOT --- var/spack/repos/builtin/packages/mkl/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py index 889851c9530..f1f07c90f5e 100644 --- a/var/spack/repos/builtin/packages/mkl/package.py +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -49,3 +49,10 @@ def setup_dependent_package(self, module, dspec): self.spec.blas_shared_lib = join_path(libdir, name) self.spec.lapack_shared_lib = self.spec.blas_shared_lib + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + # set up MKLROOT for everyone using MKL package + spack_env.set('MKLROOT', self.prefix) + + def setup_environment(self, spack_env, env): + env.set('MKLROOT', self.prefix) From b29b7595dda88831b7c0330b34df4c7b1e1c1734 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Fri, 15 Jul 2016 17:35:10 +0200 Subject: [PATCH 11/30] mkl: add info on interface and threading layers to the docstring --- var/spack/repos/builtin/packages/mkl/package.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py index f1f07c90f5e..6ea64f53133 100644 --- a/var/spack/repos/builtin/packages/mkl/package.py +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -9,7 +9,13 @@ class Mkl(IntelInstaller): Note: You will have to add the download file to a mirror so that Spack can find it. For instructions on how to set up a - mirror, see http://software.llnl.gov/spack/mirrors.html""" + mirror, see http://software.llnl.gov/spack/mirrors.html. + + To set the threading layer at run time set MKL_THREADING_LAYER + variable to one of the following values: INTEL, SEQUENTIAL, PGI. + To set interface layer at run time, use set the MKL_INTERFACE_LAYER + variable to LP64 or ILP64. + """ homepage = "https://software.intel.com/en-us/intel-mkl" From d61190c3fffc3597c49dc995590dc419b10aab73 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Sat, 16 Jul 2016 00:48:04 +0200 Subject: [PATCH 12/30] Explicitly request zmpi in module blacklist test. --- lib/spack/spack/test/modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 6d2e3705bd4..b6d35154e33 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -266,7 +266,7 @@ def test_alter_environment(self): def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist - spec = spack.spec.Spec('mpileaks') + spec = spack.spec.Spec('mpileaks ^zmpi') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) From 6175ce75596bf1abdedda3aaccb7bbed86453f0a Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 16 Jul 2016 07:37:38 +0200 Subject: [PATCH 13/30] netlib-scalapack: make sure Spack's Lapack is used --- .../repos/builtin/packages/netlib-scalapack/package.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index f7fe26a42da..cb2d8b44fc5 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -53,6 +53,15 @@ def install(self, spec, prefix): "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) ] + # Make sure we use Spack's Lapack: + options.extend([ + '-DLAPACK_FOUND=true', + '-DLAPACK_INCLUDE_DIRS=%s' % spec['lapack'].prefix.include, + '-DLAPACK_LIBRARIES=%s' % ( + spec['lapack'].lapack_shared_lib if '+shared' in spec else + spec['lapack'].lapack_static_lib), + ]) + if '+fpic' in spec: options.extend([ "-DCMAKE_C_FLAGS=-fPIC", From 0752eccfa5f865419f0ab4f3de06f5feea1aa422 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 16 Jul 2016 07:38:42 +0200 Subject: [PATCH 14/30] mumps: don't hardcode blas name; remove unused depends_on(lapack) --- var/spack/repos/builtin/packages/mumps/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 92c45c9b95d..630c33562b5 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -50,7 +50,6 @@ class Mumps(Package): depends_on('metis@5:', when='+metis') depends_on('parmetis', when="+parmetis") depends_on('blas') - depends_on('lapack') depends_on('scalapack', when='+mpi') depends_on('mpi', when='+mpi') @@ -63,7 +62,9 @@ def write_makefile_inc(self): if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') - makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + makefile_conf = ["LIBBLAS = %s" % to_link_flags( + self.spec['blas'].blas_shared_lib) + ] orderings = ['-Dpord'] @@ -189,7 +190,7 @@ def install(self, spec, prefix): install_tree('lib', prefix.lib) install_tree('include', prefix.include) - if '~mpi' in spec: + if '~mpi' in spec: lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' lib_suffix = lib_dsuffix if '+shared' in spec else '.a' install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) From 561748a063f8dab8bb5542b8bfab0878ab0fff06 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 16 Jul 2016 08:03:54 +0200 Subject: [PATCH 15/30] netlib-scalapack: flake8 fixes --- .../packages/netlib-scalapack/package.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index cb2d8b44fc5..e91c0449656 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -25,8 +25,10 @@ from spack import * import sys + class NetlibScalapack(Package): - """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" + """ScaLAPACK is a library of high-performance linear algebra routines for + parallel distributed memory machines""" homepage = "http://www.netlib.org/scalapack/" url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" @@ -48,10 +50,13 @@ class NetlibScalapack(Package): def install(self, spec, prefix): options = [ - "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), - "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), - "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) - ] + "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else + 'OFF'), + "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else + 'ON'), + # forces scalapack to use find_package(LAPACK): + "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", + ] # Make sure we use Spack's Lapack: options.extend([ @@ -75,11 +80,10 @@ def install(self, spec, prefix): make() make("install") - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly on Darwin: if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) - def setup_dependent_package(self, module, dependent_spec): spec = self.spec lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' @@ -87,4 +91,5 @@ def setup_dependent_package(self, module, dependent_spec): spec.fc_link = '-L%s -lscalapack' % spec.prefix.lib spec.cc_link = spec.fc_link - spec.libraries = [join_path(spec.prefix.lib, 'libscalapack%s' % lib_suffix)] + spec.libraries = [join_path(spec.prefix.lib, + 'libscalapack%s' % lib_suffix)] From 1bd3a702c99ec41bbed510102769bba6984bbd30 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Sat, 16 Jul 2016 08:12:57 +0200 Subject: [PATCH 16/30] mumps: flake8 fixes --- .../repos/builtin/packages/mumps/package.py | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py index 630c33562b5..b85a6d2b949 100644 --- a/var/spack/repos/builtin/packages/mumps/package.py +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -23,7 +23,10 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys +import glob + class Mumps(Package): """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" @@ -44,7 +47,6 @@ class Mumps(Package): variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') variant('shared', default=True, description='Build shared libraries') - depends_on('scotch + esmumps', when='~ptscotch+scotch') depends_on('scotch + esmumps + mpi', when='+ptscotch') depends_on('metis@5:', when='+metis') @@ -59,8 +61,8 @@ class Mumps(Package): # end before install # def patch(self): def write_makefile_inc(self): - if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: - raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: # NOQA: ignore=E501 + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') # NOQA: ignore=E501 makefile_conf = ["LIBBLAS = %s" % to_link_flags( self.spec['blas'].blas_shared_lib) @@ -70,33 +72,41 @@ def write_makefile_inc(self): if '+ptscotch' in self.spec or '+scotch' in self.spec: join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') - makefile_conf.extend( - ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, - "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, - join_lib, - join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + makefile_conf.extend([ + "ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', + 'scotch', + 'scotcherr'])) + ]) + orderings.append('-Dscotch') if '+ptscotch' in self.spec: orderings.append('-Dptscotch') if '+parmetis' in self.spec and '+metis' in self.spec: - libname = 'parmetis' if '+parmetis' in self.spec else 'metis' - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, - "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % ( + self.spec['parmetis'].prefix.lib, 'parmetis', + self.spec['metis'].prefix.lib, 'metis') + ]) orderings.append('-Dparmetis') elif '+metis' in self.spec: - makefile_conf.extend( - ["IMETIS = -I%s" % self.spec['metis'].prefix.include, - "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) + makefile_conf.extend([ + "IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis') + ]) orderings.append('-Dmetis') makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) # when building shared libs need -fPIC, otherwise - # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1' can not be used when making a shared object; recompile with -fPIC + # /usr/bin/ld: graph.o: relocation R_X86_64_32 against `.rodata.str1.1' + # can not be used when making a shared object; recompile with -fPIC fpic = '-fPIC' if '+shared' in self.spec else '' # TODO: test this part, it needs a full blas, scalapack and # partitionning environment with 64bit integers @@ -105,7 +115,7 @@ def write_makefile_inc(self): # the fortran compilation flags most probably are # working only for intel and gnu compilers this is # perhaps something the compiler should provide - ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic,'-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), + ['OPTF = %s -O -DALLOW_NON_INIT %s' % (fpic, '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8'), # NOQA: ignore=E501 'OPTL = %s -O ' % fpic, 'OPTC = %s -O -DINTSIZE64' % fpic]) else: @@ -114,7 +124,6 @@ def write_makefile_inc(self): 'OPTL = %s -O ' % fpic, 'OPTC = %s -O ' % fpic]) - if '+mpi' in self.spec: makefile_conf.extend( ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), @@ -135,16 +144,17 @@ def write_makefile_inc(self): if '+shared' in self.spec: if sys.platform == 'darwin': - # Building dylibs with mpif90 causes segfaults on 10.8 and 10.10. Use gfortran. (Homebrew) + # Building dylibs with mpif90 causes segfaults on 10.8 and + # 10.10. Use gfortran. (Homebrew) makefile_conf.extend([ 'LIBEXT=.dylib', - 'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'],prefix.lib), + 'AR=%s -dynamiclib -Wl,-install_name -Wl,%s/$(notdir $@) -undefined dynamic_lookup -o ' % (os.environ['FC'], prefix.lib), # NOQA: ignore=E501 'RANLIB=echo' ]) else: makefile_conf.extend([ 'LIBEXT=.so', - 'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib, + 'AR=$(FL) -shared -Wl,-soname -Wl,%s/$(notdir $@) -o' % prefix.lib, # NOQA: ignore=E501 'RANLIB=echo' ]) else: @@ -154,9 +164,8 @@ def write_makefile_inc(self): 'RANLIB = ranlib' ]) - - makefile_inc_template = join_path(os.path.dirname(self.module.__file__), - 'Makefile.inc') + makefile_inc_template = join_path( + os.path.dirname(self.module.__file__), 'Makefile.inc') with open(makefile_inc_template, "r") as fh: makefile_conf.extend(fh.read().split('\n')) @@ -165,8 +174,6 @@ def write_makefile_inc(self): makefile_inc = '\n'.join(makefile_conf) fh.write(makefile_inc) - - def install(self, spec, prefix): make_libs = [] @@ -194,11 +201,11 @@ def install(self, spec, prefix): lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' lib_suffix = lib_dsuffix if '+shared' in spec else '.a' install('libseq/libmpiseq%s' % lib_suffix, prefix.lib) - for f in glob.glob(join_path('libseq','*.h')): + for f in glob.glob(join_path('libseq', '*.h')): install(f, prefix.include) - # FIXME: extend the tests to mpirun -np 2 (or alike) when build with MPI - # FIXME: use something like numdiff to compare blessed output with the current + # FIXME: extend the tests to mpirun -np 2 when build with MPI + # FIXME: use something like numdiff to compare output files with working_dir('examples'): if '+float' in spec: os.system('./ssimpletest < input_simpletest_real') From 2e4966c8544efbd41d737e4ba2e7d005af32bcbb Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Mon, 18 Jul 2016 10:11:18 +0200 Subject: [PATCH 17/30] netlib-scalapack: use dso_suffix --- var/spack/repos/builtin/packages/netlib-scalapack/package.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py index e91c0449656..a96b367be40 100644 --- a/var/spack/repos/builtin/packages/netlib-scalapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -86,10 +86,9 @@ def install(self, spec, prefix): def setup_dependent_package(self, module, dependent_spec): spec = self.spec - lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' - lib_suffix = lib_dsuffix if '+shared' in spec else '.a' + lib_suffix = dso_suffix if '+shared' in spec else 'a' spec.fc_link = '-L%s -lscalapack' % spec.prefix.lib spec.cc_link = spec.fc_link spec.libraries = [join_path(spec.prefix.lib, - 'libscalapack%s' % lib_suffix)] + 'libscalapack.%s' % lib_suffix)] From 2b3ba850b31af879312bc1e7c8400c2725a5c038 Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 18 Jul 2016 12:51:58 +0100 Subject: [PATCH 18/30] Fix spack package-list to correctly handle deptypes --- lib/spack/spack/cmd/package-list.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index bc64c77eab4..ffb656c4689 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -48,7 +48,8 @@ def rst_table(elts): def print_rst_package_list(): """Print out information on all packages in restructured text.""" - pkgs = sorted(spack.repo.all_packages(), key=lambda s:s.name.lower()) + pkgs = sorted(spack.repo.all_packages(), key=lambda s: s.name.lower()) + pkg_names = [p.name for p in pkgs] print ".. _package-list:" print @@ -62,7 +63,7 @@ def print_rst_package_list(): print "Spack currently has %d mainline packages:" % len(pkgs) print - print rst_table("`%s`_" % p.name for p in pkgs) + print rst_table("`%s`_" % p for p in pkg_names) print print "-----" @@ -79,14 +80,15 @@ def print_rst_package_list(): print if pkg.versions: print "Versions:" - print " " + ", ".join(str(v) for v in reversed(sorted(pkg.versions))) + print " " + ", ".join(str(v) for v in + reversed(sorted(pkg.versions))) - for deptype in ('build', 'link', 'run'): - deps = pkg.dependencies(deptype) + for deptype in spack.alldeps: + deps = pkg.dependencies_of_type(deptype) if deps: print "%s Dependencies" % deptype.capitalize() - print " " + ", ".join("`%s`_" % d if d != "mpi" else d - for d in build_deps) + print " " + ", ".join("%s_" % d if d in pkg_names + else d for d in deps) print print "Description:" From cc027148ebdafbc58f3ad86b384c38534357062a Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 18 Jul 2016 12:52:23 +0100 Subject: [PATCH 19/30] Flake8 fixes in cmd/package-list.py file --- lib/spack/spack/cmd/package-list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index ffb656c4689..2d25ebc63ed 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -34,15 +34,15 @@ def github_url(pkg): """Link to a package file on github.""" - return ("https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" % - pkg.name) + url = "https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" # NOQA: ignore=E501 + return (url % pkg.name) def rst_table(elts): """Print out a RST-style table.""" cols = StringIO() ncol, widths = colify(elts, output=cols, tty=True) - header = " ".join("=" * (w-1) for w in widths) + header = " ".join("=" * (w - 1) for w in widths) return "%s\n%s%s" % (header, cols.getvalue(), header) From e9f42c1a94f58e5fa916705f67716c26818a7eae Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 10:21:15 -0500 Subject: [PATCH 20/30] Allow spack create to automatically detect octave build system --- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/create.py | 145 +++++++++++++++++--------------- 2 files changed, 78 insertions(+), 69 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 95bd4771ed6..9cdb0bd0125 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -59,7 +59,7 @@ def get_checksums(versions, urls, **kwargs): with Stage(url, keep=keep_stage) as stage: stage.fetch() if i == 0 and first_stage_function: - first_stage_function(stage) + first_stage_function(stage, url) hashes.append((version, spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index c9fa687b749..009b2f308b6 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -103,6 +103,64 @@ def install(self, spec, prefix): ${install} """) +# Build dependencies and extensions +dependencies_dict = { + 'autotools': "# depends_on('foo')", + 'cmake': "depends_on('cmake')", + 'scons': "depends_on('scons')", + 'python': "extends('python')", + 'R': "extends('R')", + 'octave': "extends('octave')", + 'unknown': "# depends_on('foo')" +} + +# Default installation instructions +install_dict = { + 'autotools': """\ + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'cmake': """\ + with working_dir('spack-build', create=True): + # FIXME: Modify the cmake line to suit your build system here. + cmake('..', *std_cmake_args) + + # FIXME: Add logic to build and install here. + make() + make('install')""", + + 'scons': """\ + # FIXME: Add logic to build and install here. + scons('prefix={0}'.format(prefix)) + scons('install')""", + + 'python': """\ + # FIXME: Add logic to build and install here. + python('setup.py', 'install', '--prefix={0}'.format(prefix))""", + + 'R': """\ + # FIXME: Add logic to build and install here. + R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), + self.stage.source_path)""", + + 'octave': """\ + # FIXME: Add logic to build and install here. + octave('--quiet', '--norc', + '--built-in-docstrings-file=/dev/null', + '--texi-macros-file=/dev/null', + '--eval', 'pkg prefix {0}; pkg install {1}'.format( + prefix, self.stage.archive_file))""", + + 'unknown': """\ + # FIXME: Unknown build system + make() + make('install')""" +} + def make_version_calls(ver_hash_tuples): """Adds a version() call to the package for each version found.""" @@ -133,60 +191,17 @@ def setup_parser(subparser): setup_parser.subparser = subparser -class ConfigureGuesser(object): - def __call__(self, stage): - """Try to guess the type of build system used by the project. - Set any necessary build dependencies or extensions. - Set the appropriate default installation instructions.""" +class BuildSystemGuesser(object): + def __call__(self, stage, url): + """Try to guess the type of build system used by a project based on + the contents of its archive or the URL it was downloaded from.""" - # Build dependencies and extensions - dependenciesDict = { - 'autotools': "# depends_on('foo')", - 'cmake': "depends_on('cmake', type='build')", - 'scons': "depends_on('scons', type='build')", - 'python': "extends('python', type=nolink)", - 'R': "extends('R')", - 'unknown': "# depends_on('foo')" - } - - # Default installation instructions - installDict = { - 'autotools': """\ - # FIXME: Modify the configure line to suit your build system here. - configure('--prefix={0}'.format(prefix)) - - # FIXME: Add logic to build and install here. - make() - make('install')""", - - 'cmake': """\ - with working_dir('spack-build', create=True): - # FIXME: Modify the cmake line to suit your build system here. - cmake('..', *std_cmake_args) - - # FIXME: Add logic to build and install here. - make() - make('install')""", - - 'scons': """\ - # FIXME: Add logic to build and install here. - scons('prefix={0}'.format(prefix)) - scons('install')""", - - 'python': """\ - # FIXME: Add logic to build and install here. - python('setup.py', 'install', '--prefix={0}'.format(prefix))""", - - 'R': """\ - # FIXME: Add logic to build and install here. - R('CMD', 'INSTALL', '--library={0}'.format(self.module.r_lib_dir), - self.stage.source_path)""", - - 'unknown': """\ - # FIXME: Unknown build system - make() - make('install')""" - } + # Most octave extensions are hosted on Octave-Forge: + # http://octave.sourceforge.net/index.html + # They all have the same base URL. + if 'downloads.sourceforge.net/octave/' in url: + self.build_system = 'octave' + return # A list of clues that give us an idea of the build system a package # uses. If the regular expression matches a file contained in the @@ -224,12 +239,6 @@ def __call__(self, stage): self.build_system = build_system - # Set any necessary build dependencies or extensions. - self.dependencies = dependenciesDict[build_system] - - # Set the appropriate default installation instructions - self.install = installDict[build_system] - def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL @@ -334,8 +343,8 @@ def create(parser, args): # Fetch tarballs (prompting user if necessary) versions, urls = fetch_tarballs(url, name, version) - # Try to guess what configure system is used. - guesser = ConfigureGuesser() + # Try to guess what build system is used. + guesser = BuildSystemGuesser() ver_hash_tuples = spack.cmd.checksum.get_checksums( versions, urls, first_stage_function=guesser, @@ -344,13 +353,13 @@ def create(parser, args): if not ver_hash_tuples: tty.die("Could not fetch any tarballs for %s" % name) - # Prepend 'py-' to python package names, by convention. + # Add prefix to package name if it is an extension. if guesser.build_system == 'python': - name = 'py-%s' % name - - # Prepend 'r-' to R package names, by convention. + name = 'py-{0}'.format(name) if guesser.build_system == 'R': - name = 'r-%s' % name + name = 'r-{0}'.format(name) + if guesser.build_system == 'octave': + name = 'octave-{0}'.format(name) # Create a directory for the new package. pkg_path = repo.filename_for_package_name(name) @@ -367,8 +376,8 @@ def create(parser, args): class_name=mod_to_class(name), url=url, versions=make_version_calls(ver_hash_tuples), - dependencies=guesser.dependencies, - install=guesser.install)) + dependencies=dependencies_dict[guesser.build_system], + install=install_dict[guesser.build_system])) # If everything checks out, go ahead and edit. spack.editor(pkg_path) From 199a8af7cc4e75e66cf53c3a2bb4c9b28aec64f1 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 10:39:40 -0500 Subject: [PATCH 21/30] Flake8 --- lib/spack/spack/cmd/checksum.py | 19 +++++++++++-------- lib/spack/spack/cmd/create.py | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 9cdb0bd0125..aedb0fd99cc 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -42,7 +42,8 @@ def setup_parser(subparser): '--keep-stage', action='store_true', dest='keep_stage', help="Don't clean up staging area when command completes.") subparser.add_argument( - 'versions', nargs=argparse.REMAINDER, help='Versions to generate checksums for') + 'versions', nargs=argparse.REMAINDER, + help='Versions to generate checksums for') def get_checksums(versions, urls, **kwargs): @@ -61,8 +62,8 @@ def get_checksums(versions, urls, **kwargs): if i == 0 and first_stage_function: first_stage_function(stage, url) - hashes.append((version, - spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + hashes.append((version, spack.util.crypto.checksum( + hashlib.md5, stage.archive_file))) i += 1 except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) @@ -79,12 +80,12 @@ def checksum(parser, args): # If the user asked for specific versions, use those. if args.versions: versions = {} - for v in args.versions: - v = ver(v) - if not isinstance(v, Version): + for version in args.versions: + version = ver(version) + if not isinstance(version, Version): tty.die("Cannot generate checksums for version lists or " + "version ranges. Use unambiguous versions.") - versions[v] = pkg.url_for_version(v) + versions[version] = pkg.url_for_version(version) else: versions = pkg.fetch_remote_versions() if not versions: @@ -111,5 +112,7 @@ def checksum(parser, args): if not version_hashes: tty.die("Could not fetch any versions for %s" % pkg.name) - version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] + version_lines = [ + " version('%s', '%s')" % (v, h) for v, h in version_hashes + ] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 009b2f308b6..2c440096d11 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -124,7 +124,7 @@ def install(self, spec, prefix): make() make('install')""", - 'cmake': """\ + 'cmake': """\ with working_dir('spack-build', create=True): # FIXME: Modify the cmake line to suit your build system here. cmake('..', *std_cmake_args) From 262ab401884a8b46992872b8c010fa844cb14271 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 11:33:29 -0500 Subject: [PATCH 22/30] Update build_system_guess test with new class name --- lib/spack/spack/test/__init__.py | 2 +- ...nfigure_guess.py => build_system_guess.py} | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) rename lib/spack/spack/test/{configure_guess.py => build_system_guess.py} (88%) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index fb91f247214..19bc1d89d37 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -38,7 +38,7 @@ 'directory_layout', 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', 'optional_deps', - 'make_executable', 'configure_guess', 'lock', 'database', + 'make_executable', 'build_system_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/build_system_guess.py similarity index 88% rename from lib/spack/spack/test/configure_guess.py rename to lib/spack/spack/test/build_system_guess.py index bad3673e7ad..80b867a528e 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -28,14 +28,14 @@ import unittest from llnl.util.filesystem import * -from spack.cmd.create import ConfigureGuesser +from spack.cmd.create import BuildSystemGuesser from spack.stage import Stage from spack.test.mock_packages_test import * from spack.util.executable import which class InstallTest(unittest.TestCase): - """Tests the configure guesser in spack create""" + """Tests the build system guesser in spack create""" def setUp(self): self.tar = which('tar') @@ -60,15 +60,11 @@ def check_archive(self, filename, system): with Stage(url) as stage: stage.fetch() - guesser = ConfigureGuesser() - guesser(stage) + guesser = BuildSystemGuesser() + guesser(stage, url) self.assertEqual(system, guesser.build_system) - def test_python(self): - self.check_archive('setup.py', 'python') - - def test_autotools(self): self.check_archive('configure', 'autotools') @@ -77,7 +73,17 @@ def test_cmake(self): self.check_archive('CMakeLists.txt', 'cmake') + def test_scons(self): + self.check_archive('SConstruct', 'scons') + + + def test_python(self): + self.check_archive('setup.py', 'python') + + + def test_R(self): + self.check_archive('NAMESPACE', 'R') + + def test_unknown(self): self.check_archive('foobar', 'unknown') - - From 89c9bec81ec3afa96ba25dd6aea9727abc8a8f0f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 5 Jul 2016 11:36:07 -0500 Subject: [PATCH 23/30] Flake8 --- lib/spack/spack/test/__init__.py | 19 ++++++++++--------- lib/spack/spack/test/build_system_guess.py | 8 -------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 19bc1d89d37..9f7da46a1a3 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -32,15 +32,16 @@ from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', - 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', - 'multimethod', 'install', 'package_sanity', 'config', - 'directory_layout', 'pattern', 'python_version', 'git_fetch', - 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', - 'cc', 'link_tree', 'spec_yaml', 'optional_deps', - 'make_executable', 'build_system_guess', 'lock', 'database', - 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', - 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] +test_names = [ + 'architecture', 'versions', 'url_parse', 'url_substitution', 'packages', + 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', + 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', + 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', + 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', + 'optional_deps', 'make_executable', 'build_system_guess', 'lock', + 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', + 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd' +] def list_tests(): diff --git a/lib/spack/spack/test/build_system_guess.py b/lib/spack/spack/test/build_system_guess.py index 80b867a528e..e728a47cf44 100644 --- a/lib/spack/spack/test/build_system_guess.py +++ b/lib/spack/spack/test/build_system_guess.py @@ -44,12 +44,10 @@ def setUp(self): os.chdir(self.tmpdir) self.stage = None - def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) os.chdir(self.orig_dir) - def check_archive(self, filename, system): mkdirp('archive') touch(join_path('archive', filename)) @@ -64,26 +62,20 @@ def check_archive(self, filename, system): guesser(stage, url) self.assertEqual(system, guesser.build_system) - def test_autotools(self): self.check_archive('configure', 'autotools') - def test_cmake(self): self.check_archive('CMakeLists.txt', 'cmake') - def test_scons(self): self.check_archive('SConstruct', 'scons') - def test_python(self): self.check_archive('setup.py', 'python') - def test_R(self): self.check_archive('NAMESPACE', 'R') - def test_unknown(self): self.check_archive('foobar', 'unknown') From a46138dea94cdcddd91f83aaedb7388a0e53f7e4 Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 18 Jul 2016 13:59:58 +0100 Subject: [PATCH 24/30] More flake8 fixes for package-list --- lib/spack/spack/cmd/package-list.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index 2d25ebc63ed..a27502d30e6 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -22,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import re import cgi from StringIO import StringIO -import llnl.util.tty as tty from llnl.util.tty.colify import * import spack From 2b4d2b8748669ad5a2c45d2024d265f12071c441 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 18 Jul 2016 11:39:26 -0400 Subject: [PATCH 25/30] gitignore: don't require opt to be a directory I use a symlink to make it easy to swap out. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 040df3eafd2..bfc6172a4e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /var/spack/stage /var/spack/cache *.pyc -/opt/ +/opt *~ .DS_Store .idea From 19df1ea79d5539986652de84782856bcc7a84037 Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 18 Jul 2016 17:53:26 +0200 Subject: [PATCH 26/30] flake8 fix --- lib/spack/spack/test/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index b6d35154e33..135cd028e3d 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -27,7 +27,6 @@ import StringIO import spack.modules -import unittest from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) From 49cd6f6269267b5388b3dcc31ec2cafc58567e80 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Mon, 18 Jul 2016 16:01:48 -0400 Subject: [PATCH 27/30] py-numpy: Add py-setuptools dependencies Closes #1281. --- var/spack/repos/builtin/packages/py-numpy/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index 6bc11a5e484..15adcbc2aca 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -44,6 +44,7 @@ class PyNumpy(Package): extends('python') depends_on('py-nose', type='build') + depends_on('py-setuptools') depends_on('blas', when='+blas') depends_on('lapack', when='+lapack') From 607813d5ce6dbd5063f9a0ac6d90aea8b307da3f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jul 2016 14:13:14 -0700 Subject: [PATCH 28/30] setuptools is a build dependency in numpy. --- var/spack/repos/builtin/packages/py-numpy/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py index 15adcbc2aca..2febdac6589 100644 --- a/var/spack/repos/builtin/packages/py-numpy/package.py +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -44,7 +44,7 @@ class PyNumpy(Package): extends('python') depends_on('py-nose', type='build') - depends_on('py-setuptools') + depends_on('py-setuptools', type='build') depends_on('blas', when='+blas') depends_on('lapack', when='+lapack') From 46ee4a3a5360010e77f77f201ffd5e4289516cc4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 18 Jul 2016 14:25:17 -0700 Subject: [PATCH 29/30] Using cc -craype-verbose to get cray platform type --- lib/spack/spack/platforms/cray_xc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index e710303e23a..8dc575bb716 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,6 +2,7 @@ from spack.architecture import Platform, Target from spack.operating_systems.linux_distro import LinuxDistro from spack.operating_systems.cnl import Cnl +from spack.util.executable import which class CrayXc(Platform): priority = 20 @@ -42,5 +43,11 @@ def __init__(self): @classmethod def detect(self): - return os.path.exists('/opt/cray/craype') + if os.path.exists('/cray_home'): + cc_verbose = which('cc') + cc_verbose.add_default_arg('-craype-verbose') + text = cc_verbose(output=str, error=str, ignore_errors=True).split() + if '-D__CRAYXC' in text: + return True + return False From 02340062b4d435ee97cec857f25122b8f2e3c990 Mon Sep 17 00:00:00 2001 From: mwilliammyers Date: Mon, 18 Jul 2016 16:17:07 -0600 Subject: [PATCH 30/30] package - jasper - Add new jasper package --- .../fix_alpha_channel_assert_fail.patch | 25 ++++++++ .../repos/builtin/packages/jasper/package.py | 63 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 var/spack/repos/builtin/packages/jasper/fix_alpha_channel_assert_fail.patch create mode 100644 var/spack/repos/builtin/packages/jasper/package.py diff --git a/var/spack/repos/builtin/packages/jasper/fix_alpha_channel_assert_fail.patch b/var/spack/repos/builtin/packages/jasper/fix_alpha_channel_assert_fail.patch new file mode 100644 index 00000000000..cbf79ff9719 --- /dev/null +++ b/var/spack/repos/builtin/packages/jasper/fix_alpha_channel_assert_fail.patch @@ -0,0 +1,25 @@ +diff --git a/src/libjasper/jpc/jpc_dec.c b/src/libjasper/jpc/jpc_dec.c +index fa72a0e..1f4845f 100644 +--- a/src/libjasper/jpc/jpc_dec.c ++++ b/src/libjasper/jpc/jpc_dec.c +@@ -1069,12 +1069,18 @@ static int jpc_dec_tiledecode(jpc_dec_t *dec, jpc_dec_tile_t *tile) + /* Apply an inverse intercomponent transform if necessary. */ + switch (tile->cp->mctid) { + case JPC_MCT_RCT: +- assert(dec->numcomps == 3); ++ if (dec->numcomps != 3 && dec->numcomps != 4) { ++ jas_eprintf("bad number of components (%d)\n", dec->numcomps); ++ return -1; ++ } + jpc_irct(tile->tcomps[0].data, tile->tcomps[1].data, + tile->tcomps[2].data); + break; + case JPC_MCT_ICT: +- assert(dec->numcomps == 3); ++ if (dec->numcomps != 3 && dec->numcomps != 4) { ++ jas_eprintf("bad number of components (%d)\n", dec->numcomps); ++ return -1; ++ } + jpc_iict(tile->tcomps[0].data, tile->tcomps[1].data, + tile->tcomps[2].data); + break; diff --git a/var/spack/repos/builtin/packages/jasper/package.py b/var/spack/repos/builtin/packages/jasper/package.py new file mode 100644 index 00000000000..f450c7d155f --- /dev/null +++ b/var/spack/repos/builtin/packages/jasper/package.py @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Jasper(Package): + """Library for manipulating JPEG-2000 images""" + + homepage = "https://www.ece.uvic.ca/~frodo/jasper/" + url = "https://www.ece.uvic.ca/~frodo/jasper/software/jasper-1.900.1.zip" + + version('1.900.1', 'a342b2b4495b3e1394e161eb5d85d754') + + variant('shared', default=True, + description='Builds shared versions of the libraries') + variant('debug', default=False, + description='Builds debug versions of the libraries') + + depends_on('libjpeg-turbo') + + # Fixes a bug (still in upstream as of v.1.900.1) where an assertion fails + # when certain JPEG-2000 files with an alpha channel are processed + # see: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=469786 + patch('fix_alpha_channel_assert_fail.patch') + + def install(self, spec, prefix): + configure_options = [ + '--prefix={0}'.format(prefix), + '--mandir={0}'.format(spec.prefix.man), + ] + + if '+shared' in spec: + configure_options.append('--enable-shared') + + if '+debug' not in spec: + configure_options.append('--disable-debug') + + configure(*configure_options) + + make() + make('install')