zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into pkg-graphviz

Browse Source
This commit is contained in:
Kelly (KT) Thompson 2016-03-21 16:24:17 -06:00
commit 799f1f6768
13 changed files with 221 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
lib/spack/docs/packaging_guide.rst
View File

@ -1844,6 +1844,20 @@ dedicated process.
.. _prefix-objects:
Failing the build
----------------------
Sometimes you don't want a package to successfully install unless some
condition is true. You can explicitly cause the build to fail from
``install()`` by raising an ``InstallError``, for example:
.. code-block:: python
if spec.architecture.startswith('darwin'):
raise InstallError('This package does not build on Mac OS X!')
Prefix objects
----------------------
@ -2160,6 +2174,62 @@ package, this allows us to avoid race conditions in the library's
build system.
.. _sanity-checks:
Sanity checking an intallation
--------------------------------
By default, Spack assumes that a build has failed if nothing is
written to the install prefix, and that it has succeeded if anything
(a file, a directory, etc.) is written to the install prefix after
``install()`` completes.
Consider a simple autotools build like this:
.. code-block:: python
def install(self, spec, prefix):
configure("--prefix=" + prefix)
make()
make("install")
If you are using using standard autotools or CMake, ``configure`` and
``make`` will not write anything to the install prefix. Only ``make
install`` writes the files, and only once the build is already
complete. Not all builds are like this. Many builds of scientific
software modify the install prefix *before* ``make install``. Builds
like this can falsely report that they were successfully installed if
an error occurs before the install is complete but after files have
been written to the ``prefix``.
``sanity_check_is_file`` and ``sanity_check_is_dir``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can optionally specify *sanity checks* to deal with this problem.
Add properties like this to your package:
.. code-block:: python
class MyPackage(Package):
...
sanity_check_is_file = ['include/libelf.h']
sanity_check_is_dir = [lib]
def install(self, spec, prefix):
configure("--prefix=" + prefix)
make()
make("install")
Now, after ``install()`` runs, Spack will check whether
``$prefix/include/libelf.h`` exists and is a file, and whether
``$prefix/lib`` exists and is a directory. If the checks fail, then
the build will fail and the install prefix will be removed. If they
succeed, Spack considers the build succeeful and keeps the prefix in
place.
.. _file-manipulation:
File manipulation functions

7
lib/spack/spack/__init__.py
View File

@ -188,3 +188,10 @@
import spack.util.executable
from spack.util.executable import *
__all__ += spack.util.executable.__all__
from spack.package import \
install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \
InstallError, ExternalPackageError
__all__ += [
'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError',
'InstallError', 'ExternalPackageError']

3
lib/spack/spack/compilers/gcc.py
View File

@ -40,7 +40,8 @@ class Gcc(Compiler):
fc_names = ['gfortran']
# MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
suffixes = [r'-mp-\d\.\d']
# Homebrew and Linuxes may build gcc with -X, -X.Y suffixes
suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d']
# Named wrapper links within spack.build_env_path
link_paths = {'cc' : 'gcc/gcc',

55
lib/spack/spack/package.py
View File

@ -318,16 +318,17 @@ class SomePackage(Package):
"""Most packages are NOT extendable. Set to True if you want extensions."""
extendable = False
"""List of prefix-relative file paths. If these do not exist after
install, or if they exist but are not files, sanity checks fail.
"""List of prefix-relative file paths (or a single path). If these do
not exist after install, or if they exist but are not files,
sanity checks fail.
"""
sanity_check_files = []
sanity_check_is_file = []
"""List of prefix-relative directory paths. If these do not exist
after install, or if they exist but are not directories, sanity
checks will fail.
"""List of prefix-relative directory paths (or a single path). If
these do not exist after install, or if they exist but are not
directories, sanity checks will fail.
"""
sanity_check_dirs = []
sanity_check_is_dir = []
def __init__(self, spec):
@ -966,14 +967,17 @@ def build_process():
def sanity_check_prefix(self):
"""This function checks whether install succeeded."""
def check_paths(path_list, filetype, predicate):
if isinstance(path_list, basestring):
path_list = [path_list]
for path in path_list:
abs_path = os.path.join(self.prefix, path)
if not predicate(abs_path):
raise InstallError("Install failed for %s. No such %s in prefix: %s"
% (self.name, filetype, path))
check_paths(self.sanity_check_files, 'file', os.path.isfile)
check_paths(self.sanity_check_dirs, 'directory', os.path.isdir)
check_paths(self.sanity_check_is_file, 'file', os.path.isfile)
check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir)
installed = set(os.listdir(self.prefix))
installed.difference_update(spack.install_layout.hidden_file_paths)
@ -1239,6 +1243,27 @@ def rpath_args(self):
return " ".join("-Wl,-rpath,%s" % p for p in self.rpath)
def install_dependency_symlinks(pkg, spec, prefix):
"""Execute a dummy install and flatten dependencies"""
flatten_dependencies(spec, prefix)
def flatten_dependencies(spec, flat_dir):
"""Make each dependency of spec present in dir via symlink."""
for dep in spec.traverse(root=False):
name = dep.name
dep_path = spack.install_layout.path_for_spec(dep)
dep_files = LinkTree(dep_path)
os.mkdir(flat_dir+'/'+name)
conflict = dep_files.find_conflict(flat_dir+'/'+name)
if conflict:
raise DependencyConflictError(conflict)
dep_files.merge(flat_dir+'/'+name)
def validate_package_url(url_string):
"""Determine whether spack can handle a particular URL or not."""
url = urlparse(url_string)
@ -1326,6 +1351,10 @@ def __init__(self, message, long_msg=None):
super(InstallError, self).__init__(message, long_msg)
class ExternalPackageError(InstallError):
"""Raised by install() when a package is only for external use."""
class PackageStillNeededError(InstallError):
"""Raised when package is still needed by another on uninstall."""
def __init__(self, spec, dependents):
@ -1376,3 +1405,11 @@ def __init__(self, path):
class ActivationError(ExtensionError):
def __init__(self, msg, long_msg=None):
super(ActivationError, self).__init__(msg, long_msg)
class DependencyConflictError(spack.error.SpackError):
"""Raised when the dependencies cannot be flattened as asked for."""
def __init__(self, conflict):
super(DependencyConflictError, self).__init__(
"%s conflicts with another file in the flattened directory." %(
conflict))

2
lib/spack/spack/url.py
View File

@ -142,7 +142,7 @@ def split_url_extension(path):
def downloaded_file_extension(path):
"""This returns the type of archive a URL refers to. This is
sometimes confusing becasue of URLs like:
sometimes confusing because of URLs like:
(1) https://github.com/petdance/ack/tarball/1.93_02

7
lib/spack/spack/util/compression.py
View File

@ -27,13 +27,12 @@
from itertools import product
from spack.util.executable import which
# Supported archvie extensions.
# Supported archive extensions.
PRE_EXTS = ["tar"]
EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
# Add EXTS last so that .tar.gz is matched *before* tar.gz
ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz
ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS
def allowed_archive(path):
return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES)

2
share/spack/setup-env.sh
View File

@ -141,7 +141,7 @@ function _spack_pathadd {
fi
# Do the actual prepending here.
eval "_pa_oldvalue=\$${_pa_varname}"
eval "_pa_oldvalue=\${${_pa_varname}:-}"
if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then
if [ -n "$_pa_oldvalue" ]; then

1
var/spack/repos/builtin/packages/curl/package.py
View File

@ -7,6 +7,7 @@ class Curl(Package):
homepage = "http://curl.haxx.se"
url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2"
version('7.47.1', '9ea3123449439bbd960cd25cf98796fb')
version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148')
version('7.45.0', '62c1a352b28558f25ba6209214beadc8')
version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae')

3
var/spack/repos/builtin/packages/libelf/package.py
View File

@ -38,8 +38,7 @@ class Libelf(Package):
provides('elf')
sanity_check_files = ['include/libelf.h']
sanity_check_dirs = ['lib']
sanity_check_is_file = 'include/libelf.h'
def install(self, spec, prefix):
configure("--prefix=" + prefix,

5
var/spack/repos/builtin/packages/netlib-scalapack/package.py
View File

@ -1,4 +1,5 @@
from spack import *
import sys
class NetlibScalapack(Package):
"""ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines"""
@ -41,8 +42,8 @@ def install(self, spec, prefix):
make("install")
def setup_dependent_environment(self, module, spec, dependent_spec):
# TODO treat OS that are not Linux...
lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a'
lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so'
lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a'
spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib
spec['scalapack'].cc_link = spec['scalapack'].fc_link

7
var/spack/repos/builtin/packages/openblas/package.py
View File

@ -1,10 +1,12 @@
from spack import *
import sys
class Openblas(Package):
"""OpenBLAS: An optimized BLAS library"""
homepage = "http://www.openblas.net"
url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc')
version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
# virtual dependency
@ -15,13 +17,14 @@ def install(self, spec, prefix):
make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77')
make('install', "PREFIX='%s'" % prefix)
lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so'
# Blas virtual package should provide blas.a and libblas.a
with working_dir(prefix.lib):
symlink('libopenblas.a', 'blas.a')
symlink('libopenblas.a', 'libblas.a')
symlink('libopenblas.so', 'libblas.so')
symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix)
# Lapack virtual package should provide liblapack.a
with working_dir(prefix.lib):
symlink('libopenblas.a', 'liblapack.a')
symlink('libopenblas.so', 'liblapack.so')
symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix)

1
var/spack/repos/builtin/packages/papi/package.py
View File

@ -13,6 +13,7 @@ class Papi(Package):
homepage = "http://icl.cs.utk.edu/papi/index.html"
url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz"
version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412')
version('5.4.1', '9134a99219c79767a11463a76b0b01a2')
version('5.3.0', '367961dd0ab426e5ae367c2713924ffb')

63
var/spack/repos/builtin/packages/superlu-dist/package.py Normal file
View File

@ -0,0 +1,63 @@
from spack import *
class SuperluDist(Package):
"""A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines."""
homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/"
url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz"
version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae')
version('4.2', 'ae9fafae161f775fbac6eba11e530a65')
version('4.1', '4edee38cc29f687bd0c8eb361096a455')
version('4.0', 'c0b98b611df227ae050bc1635c6940e0')
depends_on ('mpi')
depends_on ('blas')
depends_on ('lapack')
depends_on ('parmetis')
depends_on ('metis')
def install(self, spec, prefix):
makefile_inc = []
makefile_inc.extend([
'PLAT = _mac_x',
'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix
'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a',
'BLASDEF = -DUSE_VENDOR_BLAS',
'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names
'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib,
'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib,
'FLIBS =',
'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)',
'ARCH = ar',
'ARCHFLAGS = cr',
'RANLIB = true',
'CC = mpicc', # FIXME avoid hardcoding MPI compiler names
'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include),
'NOOPTS = -fPIC -std=c99',
'FORTRAN = mpif77',
'F90FLAGS = -O2',
'LOADER = mpif77',
'LOADOPTS =',
'CDEFS = -DAdd_'
])
#with working_dir('src'):
with open('make.inc', 'w') as fh:
fh.write('\n'.join(makefile_inc))
make("lib", parallel=False)
# FIXME:
# cd "EXAMPLE" do
# system "make"
# need to install by hand
headers_location = join_path(self.prefix.include,'superlu_dist')
mkdirp(headers_location)
# FIXME: fetch all headers in the folder automatically
for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']:
superludist_header = join_path(self.stage.source_path, 'SRC/',header)
install(superludist_header, headers_location)
superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a')
install(superludist_lib,self.prefix.lib)