diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 035f9a13ff4..25dc7474998 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,27 +25,32 @@ import collections import errno import fileinput +import fnmatch import glob import numbers import os import re import shutil +import six import stat import subprocess import sys from contextlib import contextmanager -import llnl.util.tty as tty +from llnl.util import tty from llnl.util.lang import dedupe __all__ = [ 'FileFilter', + 'HeaderList', 'LibraryList', 'ancestor', 'can_access', 'change_sed_delimiter', 'copy_mode', 'filter_file', + 'find', + 'find_headers', 'find_libraries', 'find_system_libraries', 'fix_darwin_install_name', @@ -66,25 +71,32 @@ 'touchp', 'traverse_tree', 'unset_executable_mode', - 'working_dir'] + 'working_dir' +] def filter_file(regex, repl, *filenames, **kwargs): - """Like sed, but uses python regular expressions. + r"""Like sed, but uses python regular expressions. - Filters every line of each file through regex and replaces the file - with a filtered version. Preserves mode of filtered files. + Filters every line of each file through regex and replaces the file + with a filtered version. Preserves mode of filtered files. - As with re.sub, ``repl`` can be either a string or a callable. - If it is a callable, it is passed the match object and should - return a suitable replacement string. If it is a string, it - can contain ``\1``, ``\2``, etc. to represent back-substitution - as sed would allow. + As with re.sub, ``repl`` can be either a string or a callable. + If it is a callable, it is passed the match object and should + return a suitable replacement string. If it is a string, it + can contain ``\1``, ``\2``, etc. to represent back-substitution + as sed would allow. - Keyword Options: - string[=False] If True, treat regex as a plain string. - backup[=True] Make backup file(s) suffixed with ~ - ignore_absent[=False] Ignore any files that don't exist. + Parameters: + regex (str): The regular expression to search for + repl (str): The string to replace matches with + *filenames: One or more files to search and replace + + Keyword Arguments: + string (bool): Treat regex as a plain string. Default it False + backup (bool): Make backup file(s) suffixed with ``~``. Default is True + ignore_absent (bool): Ignore any files that don't exist. + Default is False """ string = kwargs.get('string', False) backup = kwargs.get('backup', True) @@ -128,7 +140,7 @@ def groupid_to_group(x): class FileFilter(object): - """Convenience class for calling filter_file a lot.""" + """Convenience class for calling ``filter_file`` a lot.""" def __init__(self, *filenames): self.filenames = filenames @@ -139,12 +151,18 @@ def filter(self, regex, repl, **kwargs): def change_sed_delimiter(old_delim, new_delim, *filenames): """Find all sed search/replace commands and change the delimiter. - e.g., if the file contains seds that look like 's///', you can - call change_sed_delimiter('/', '@', file) to change the - delimiter to '@'. - NOTE that this routine will fail if the delimiter is ' or ". - Handling those is left for future work. + e.g., if the file contains seds that look like ``'s///'``, you can + call ``change_sed_delimiter('/', '@', file)`` to change the + delimiter to ``'@'``. + + Note that this routine will fail if the delimiter is ``'`` or ``"``. + Handling those is left for future work. + + Parameters: + old_delim (str): The delimiter to search for + new_delim (str): The delimiter to replace with + *filenames: One or more files to search and replace """ assert(len(old_delim) == 1) assert(len(new_delim) == 1) @@ -239,7 +257,7 @@ def mkdirp(*paths): def force_remove(*paths): - """Remove files without printing errors. Like rm -f, does NOT + """Remove files without printing errors. Like ``rm -f``, does NOT remove directories.""" for path in paths: try: @@ -278,7 +296,8 @@ def touch(path): def touchp(path): - """Like touch, but creates any parent directories needed for the file.""" + """Like ``touch``, but creates any parent directories needed for the file. + """ mkdirp(os.path.dirname(path)) touch(path) @@ -335,17 +354,13 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): ('root/b', 'dest/b') ('root/b/file3', 'dest/b/file3') - Optional args: - - order=[pre|post] -- Whether to do pre- or post-order traversal. - - ignore= -- Predicate indicating which files to ignore. - - follow_nonexisting -- Whether to descend into directories in - src that do not exit in dest. Default True. - - follow_links -- Whether to descend into symlinks in src. - + Keyword Arguments: + order (str): Whether to do pre- or post-order traversal. Accepted + values are 'pre' and 'post' + ignore (str): Predicate indicating which files to ignore + follow_nonexisting (bool): Whether to descend into directories in + ``src`` that do not exit in ``dest``. Default is True + follow_links (bool): Whether to descend into symlinks in ``src`` """ follow_nonexisting = kwargs.get('follow_nonexisting', True) follow_links = kwargs.get('follow_link', False) @@ -406,12 +421,10 @@ def set_executable(path): def remove_dead_links(root): - """ - Removes any dead link that is present in root - - Args: - root: path where to search for dead links + """Removes any dead link that is present in root. + Parameters: + root (str): path where to search for dead links """ for file in os.listdir(root): path = join_path(root, file) @@ -419,11 +432,10 @@ def remove_dead_links(root): def remove_if_dead_link(path): - """ - Removes the argument if it is a dead link, does nothing otherwise + """Removes the argument if it is a dead link. - Args: - path: the potential dead link + Parameters: + path (str): The potential dead link """ if os.path.islink(path): real_path = os.path.realpath(path) @@ -432,14 +444,13 @@ def remove_if_dead_link(path): def remove_linked_tree(path): - """ - Removes a directory and its contents. If the directory is a - symlink, follows the link and removes the real directory before - removing the link. + """Removes a directory and its contents. - Args: - path: directory to be removed + If the directory is a symlink, follows the link and removes the real + directory before removing the link. + Parameters: + path (str): Directory to be removed """ if os.path.exists(path): if os.path.islink(path): @@ -450,17 +461,17 @@ def remove_linked_tree(path): def fix_darwin_install_name(path): - """ - Fix install name of dynamic libraries on Darwin to have full path. + """Fix install name of dynamic libraries on Darwin to have full path. + There are two parts of this task: - (i) use install_name('-id',...) to change install name of a single lib; - (ii) use install_name('-change',...) to change the cross linking between - libs. The function assumes that all libraries are in one folder and - currently won't follow subfolders. - Args: - path: directory in which .dylib files are located + 1. Use ``install_name('-id', ...)`` to change install name of a single lib + 2. Use ``install_name('-change', ...)`` to change the cross linking between + libs. The function assumes that all libraries are in one folder and + currently won't follow subfolders. + Parameters: + path (str): directory in which .dylib files are located """ libs = glob.glob(join_path(path, "*.dylib")) for lib in libs: @@ -486,29 +497,108 @@ def fix_darwin_install_name(path): stdout=subprocess.PIPE).communicate()[0] break -# Utilities for libraries + +def find(root, files, recurse=True): + """Search for ``files`` starting from the ``root`` directory. + + Like GNU/BSD find but written entirely in Python. + + Examples: + + .. code-block:: console + + $ find /usr -name python + + is equivalent to: + + >>> find('/usr', 'python') + + .. code-block:: console + + $ find /usr/local/bin -maxdepth 1 -name python + + is equivalent to: + + >>> find('/usr/local/bin', 'python', recurse=False) + + Accepts any glob characters accepted by fnmatch: + + ======= ==================================== + Pattern Meaning + ======= ==================================== + * matches everything + ? matches any single character + [seq] matches any character in ``seq`` + [!seq] matches any character not in ``seq`` + ======= ==================================== + + Parameters: + root (str): The root directory to start searching from + files (str or collections.Sequence): Library name(s) to search for + recurse (bool, optional): if False search only root folder, + if True descends top-down from the root. Defaults to True. + + Returns: + :func:`list`: The files that have been found + """ + if isinstance(files, six.string_types): + files = [files] + + if recurse: + return _find_recursive(root, files) + else: + return _find_non_recursive(root, files) -class LibraryList(collections.Sequence): - """Sequence of absolute paths to libraries +def _find_recursive(root, search_files): + found_files = [] - Provides a few convenience methods to manipulate library paths and get - commonly used compiler flags or names + for path, _, list_files in os.walk(root): + for search_file in search_files: + for list_file in list_files: + if fnmatch.fnmatch(list_file, search_file): + found_files.append(join_path(path, list_file)) + + return found_files + + +def _find_non_recursive(root, search_files): + found_files = [] + + for list_file in os.listdir(root): + for search_file in search_files: + if fnmatch.fnmatch(list_file, search_file): + found_files.append(join_path(root, list_file)) + + return found_files + + +# Utilities for libraries and headers + + +class FileList(collections.Sequence): + """Sequence of absolute paths to files. + + Provides a few convenience methods to manipulate file paths. """ - def __init__(self, libraries): - self.libraries = list(libraries) + def __init__(self, files): + if isinstance(files, six.string_types): + files = [files] + + self.files = list(dedupe(files)) @property def directories(self): - """Stable de-duplication of the directories where the libraries - reside + """Stable de-duplication of the directories where the files reside. >>> l = LibraryList(['/dir1/liba.a', '/dir2/libb.a', '/dir1/libc.a']) >>> assert l.directories == ['/dir1', '/dir2'] + >>> h = HeaderList(['/dir1/a.h', '/dir1/b.h', '/dir2/c.h']) + >>> assert h.directories == ['/dir1', '/dir2'] """ return list(dedupe( - os.path.dirname(x) for x in self.libraries if os.path.dirname(x) + os.path.dirname(x) for x in self.files if os.path.dirname(x) )) @property @@ -517,8 +607,150 @@ def basenames(self): >>> l = LibraryList(['/dir1/liba.a', '/dir2/libb.a', '/dir3/liba.a']) >>> assert l.basenames == ['liba.a', 'libb.a'] + >>> h = HeaderList(['/dir1/a.h', '/dir2/b.h', '/dir3/a.h']) + >>> assert h.basenames == ['a.h', 'b.h'] """ - return list(dedupe(os.path.basename(x) for x in self.libraries)) + return list(dedupe(os.path.basename(x) for x in self.files)) + + @property + def names(self): + """Stable de-duplication of file names in the list + + >>> h = HeaderList(['/dir1/a.h', '/dir2/b.h', '/dir3/a.h']) + >>> assert h.names == ['a', 'b'] + """ + return list(dedupe(x.split('.')[0] for x in self.basenames)) + + def __getitem__(self, item): + cls = type(self) + if isinstance(item, numbers.Integral): + return self.files[item] + return cls(self.files[item]) + + def __add__(self, other): + return self.__class__(dedupe(self.files + list(other))) + + def __radd__(self, other): + return self.__add__(other) + + def __eq__(self, other): + return self.files == other.files + + def __len__(self): + return len(self.files) + + def joined(self, separator=' '): + return separator.join(self.files) + + def __repr__(self): + return self.__class__.__name__ + '(' + repr(self.files) + ')' + + def __str__(self): + return self.joined() + + +class HeaderList(FileList): + """Sequence of absolute paths to headers. + + Provides a few convenience methods to manipulate header paths and get + commonly used compiler flags or names. + """ + + def __init__(self, files): + super(HeaderList, self).__init__(files) + + self._macro_definitions = [] + + @property + def headers(self): + return self.files + + @property + def include_flags(self): + """Include flags + + >>> h = HeaderList(['/dir1/a.h', '/dir1/b.h', '/dir2/c.h']) + >>> assert h.cpp_flags == '-I/dir1 -I/dir2' + """ + return ' '.join(['-I' + x for x in self.directories]) + + @property + def macro_definitions(self): + """Macro definitions + + >>> h = HeaderList(['/dir1/a.h', '/dir1/b.h', '/dir2/c.h']) + >>> h.add_macro('-DBOOST_LIB_NAME=boost_regex') + >>> h.add_macro('-DBOOST_DYN_LINK') + >>> assert h.macro_definitions == '-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK' # noqa + """ + return ' '.join(self._macro_definitions) + + @property + def cpp_flags(self): + """Include flags + macro definitions + + >>> h = HeaderList(['/dir1/a.h', '/dir1/b.h', '/dir2/c.h']) + >>> h.add_macro('-DBOOST_DYN_LINK') + >>> assert h.macro_definitions == '-I/dir1 -I/dir2 -DBOOST_DYN_LINK' + """ + return self.include_flags + ' ' + self.macro_definitions + + def add_macro(self, macro): + """Add a macro definition""" + self._macro_definitions.append(macro) + + +def find_headers(headers, root, recurse=False): + """Returns an iterable object containing a list of full paths to + headers if found. + + Accepts any glob characters accepted by fnmatch: + + ======= ==================================== + Pattern Meaning + ======= ==================================== + * matches everything + ? matches any single character + [seq] matches any character in ``seq`` + [!seq] matches any character not in ``seq`` + ======= ==================================== + + Parameters: + headers (str or list of str): Header name(s) to search for + root (str): The root directory to start searching from + recurses (bool, optional): if False search only root folder, + if True descends top-down from the root. Defaults to False. + + Returns: + HeaderList: The headers that have been found + """ + if isinstance(headers, six.string_types): + headers = [headers] + elif not isinstance(headers, collections.Sequence): + message = '{0} expects a string or sequence of strings as the ' + message += 'first argument [got {1} instead]' + message = message.format(find_headers.__name__, type(headers)) + raise TypeError(message) + + # Construct the right suffix for the headers + suffix = 'h' + + # List of headers we are searching with suffixes + headers = ['{0}.{1}'.format(header, suffix) for header in headers] + + return HeaderList(find(root, headers, recurse)) + + +class LibraryList(FileList): + """Sequence of absolute paths to libraries + + Provides a few convenience methods to manipulate library paths and get + commonly used compiler flags or names + """ + + @property + def libraries(self): + return self.files @property def names(self): @@ -556,36 +788,9 @@ def ld_flags(self): """ return self.search_flags + ' ' + self.link_flags - def __getitem__(self, item): - cls = type(self) - if isinstance(item, numbers.Integral): - return self.libraries[item] - return cls(self.libraries[item]) - def __add__(self, other): - return LibraryList(dedupe(self.libraries + list(other))) - - def __radd__(self, other): - return self.__add__(other) - - def __eq__(self, other): - return self.libraries == other.libraries - - def __len__(self): - return len(self.libraries) - - def joined(self, separator=' '): - return separator.join(self.libraries) - - def __repr__(self): - return self.__class__.__name__ + '(' + repr(self.libraries) + ')' - - def __str__(self): - return self.joined() - - -def find_system_libraries(library_names, shared=True): - """Searches the usual system library locations for ``library_names``. +def find_system_libraries(libraries, shared=True): + """Searches the usual system library locations for ``libraries``. Search order is as follows: @@ -596,20 +801,32 @@ def find_system_libraries(library_names, shared=True): 5. ``/usr/local/lib64`` 6. ``/usr/local/lib`` - Args: - library_names (str or list of str): Library name(s) to search for - shared (bool): searches for shared libraries if True + Accepts any glob characters accepted by fnmatch: + + ======= ==================================== + Pattern Meaning + ======= ==================================== + * matches everything + ? matches any single character + [seq] matches any character in ``seq`` + [!seq] matches any character not in ``seq`` + ======= ==================================== + + Parameters: + libraries (str or list of str): Library name(s) to search for + shared (bool, optional): if True searches for shared libraries, + otherwise for static. Defaults to True. Returns: LibraryList: The libraries that have been found """ - if isinstance(library_names, str): - library_names = [library_names] - elif not isinstance(library_names, collections.Sequence): + if isinstance(libraries, six.string_types): + libraries = [libraries] + elif not isinstance(libraries, collections.Sequence): message = '{0} expects a string or sequence of strings as the ' message += 'first argument [got {1} instead]' - message = message.format( - find_system_libraries.__name__, type(library_names)) + message = message.format(find_system_libraries.__name__, + type(libraries)) raise TypeError(message) libraries_found = [] @@ -622,7 +839,7 @@ def find_system_libraries(library_names, shared=True): '/usr/local/lib', ] - for library in library_names: + for library in libraries: for root in search_locations: result = find_libraries(library, root, shared, recurse=True) if result: @@ -632,26 +849,38 @@ def find_system_libraries(library_names, shared=True): return libraries_found -def find_libraries(library_names, root, shared=True, recurse=False): +def find_libraries(libraries, root, shared=True, recurse=False): """Returns an iterable of full paths to libraries found in a root dir. - Args: - library_names (str or list of str): Library names to search for + Accepts any glob characters accepted by fnmatch: + + ======= ==================================== + Pattern Meaning + ======= ==================================== + * matches everything + ? matches any single character + [seq] matches any character in ``seq`` + [!seq] matches any character not in ``seq`` + ======= ==================================== + + Parameters: + libraries (str or list of str): Library name(s) to search for root (str): The root directory to start searching from - shared (bool): if True searches for shared libraries, otherwise static. - recurse (bool): if False search only root folder, - if True descends top-down from the root + shared (bool, optional): if True searches for shared libraries, + otherwise for static. Defaults to True. + recurse (bool, optional): if False search only root folder, + if True descends top-down from the root. Defaults to False. Returns: LibraryList: The libraries that have been found """ - if isinstance(library_names, str): - library_names = [library_names] - elif not isinstance(library_names, collections.Sequence): + if isinstance(libraries, six.string_types): + libraries = [libraries] + elif not isinstance(libraries, collections.Sequence): message = '{0} expects a string or sequence of strings as the ' message += 'first argument [got {1} instead]' - raise TypeError(message.format( - find_libraries.__name__, type(library_names))) + message = message.format(find_libraries.__name__, type(libraries)) + raise TypeError(message) # Construct the right suffix for the library if shared is True: @@ -659,38 +888,6 @@ def find_libraries(library_names, root, shared=True, recurse=False): else: suffix = 'a' # List of libraries we are searching with suffixes - libraries = ['{0}.{1}'.format(lib, suffix) for lib in library_names] - # Search method - if recurse is False: - search_method = _find_libraries_non_recursive - else: - search_method = _find_libraries_recursive + libraries = ['{0}.{1}'.format(lib, suffix) for lib in libraries] - return search_method(libraries, root) - - -def _find_libraries_recursive(libraries, root): - library_dict = collections.defaultdict(list) - for path, _, files in os.walk(root): - for lib in libraries: - if lib in files: - library_dict[lib].append( - join_path(path, lib) - ) - answer = [] - for lib in libraries: - answer.extend(library_dict[lib]) - return LibraryList(answer) - - -def _find_libraries_non_recursive(libraries, root): - - def lib_or_none(lib): - library = join_path(root, lib) - if not os.path.exists(library): - return None - return library - - return LibraryList( - [lib_or_none(lib) for lib in libraries if lib_or_none(lib) is not None] - ) + return LibraryList(find(root, libraries, recurse)) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1e2210f928a..12b91bcdb01 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -64,6 +64,7 @@ from llnl.util.link_tree import LinkTree from llnl.util.tty.log import log_output from spack import directory_layout +from spack.util.executable import which from spack.stage import Stage, ResourceStage, StageComposite from spack.util.environment import dump_environment from spack.version import * @@ -1025,17 +1026,27 @@ def namespace(self): return namespace def do_fake_install(self): - """Make a fake install directory containing a 'fake' file in bin.""" - # FIXME : Make this part of the 'install' behavior ? - mkdirp(self.prefix.bin) - touch(join_path(self.prefix.bin, 'fake')) - mkdirp(self.prefix.include) - mkdirp(self.prefix.lib) + """Make a fake install directory containing fake executables, + headers, and libraries.""" + + name = self.name library_name = 'lib' + self.name - dso_suffix = 'dylib' if sys.platform == 'darwin' else 'so' + dso_suffix = '.dylib' if sys.platform == 'darwin' else '.so' + chmod = which('chmod') + + mkdirp(self.prefix.bin) + touch(join_path(self.prefix.bin, name)) + chmod('+x', join_path(self.prefix.bin, name)) + + mkdirp(self.prefix.include) + touch(join_path(self.prefix.include, name + '.h')) + + mkdirp(self.prefix.lib) touch(join_path(self.prefix.lib, library_name + dso_suffix)) touch(join_path(self.prefix.lib, library_name + '.a')) + mkdirp(self.prefix.man1) + packages_dir = spack.store.layout.build_packages_path(self.spec) dump_packages(self.spec, packages_dir) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4cd89f59bfc..0d8fb2893b8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -101,6 +101,8 @@ import ctypes import hashlib import itertools +import os + from operator import attrgetter from six import StringIO from six import string_types @@ -115,12 +117,13 @@ import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml -from llnl.util.filesystem import find_libraries +from llnl.util.filesystem import find_headers, find_libraries, is_exe from llnl.util.lang import * from llnl.util.tty.color import * from spack.build_environment import get_path_from_module, load_module from spack.provider_index import ProviderIndex from spack.util.crypto import prefix_bits +from spack.util.executable import Executable from spack.util.prefix import Prefix from spack.util.spack_yaml import syaml_dict from spack.util.string import * @@ -745,9 +748,9 @@ def __str__(self): class DependencyMap(HashableMap): - """Each spec has a DependencyMap containing specs for its dependencies. The DependencyMap is keyed by name. """ + @property def concrete(self): return all((d.spec.concrete and d.deptypes) @@ -757,38 +760,104 @@ def __str__(self): return "{deps: %s}" % ', '.join(str(d) for d in sorted(self.values())) -def _libs_default_handler(descriptor, spec, cls): - """Default handler when for ``libs`` attribute in Spec interface. +def _command_default_handler(descriptor, spec, cls): + """Default handler when looking for the 'command' attribute. - Tries to search for ``lib{spec.name}`` recursively starting from - `spec.prefix`. + Tries to search for ``spec.name`` in the ``spec.prefix.bin`` directory. - Args: - descriptor (ForwardQueryToPackage): descriptor that triggered - the call + Parameters: + descriptor (ForwardQueryToPackage): descriptor that triggered the call spec (Spec): spec that is being queried - cls (type(spec)): type of spec, to match the signature of the - descriptor `__get__` method - """ - name = 'lib' + spec.name - shared = '+shared' in spec - return find_libraries(name, root=spec.prefix, shared=shared, recurse=True) - - -def _cppflags_default_handler(descriptor, spec, cls): - """Default handler for the ``cppflags`` attribute in the Spec interface. - - The default just returns ``-I{spec.prefix.include}``. - - Args: - descriptor (ForwardQueryToPackage): descriptor that triggered - the call - spec (Spec): spec that is being queried - cls (type(spec)): type of spec, to match the signature of the descriptor ``__get__`` method + + Returns: + Executable: An executable of the command + + Raises: + RuntimeError: If the command is not found """ - return '-I' + spec.prefix.include + path = os.path.join(spec.prefix.bin, spec.name) + + if is_exe(path): + return Executable(path) + else: + msg = 'Unable to locate {0} command in {1}' + raise RuntimeError(msg.format(spec.name, spec.prefix.bin)) + + +def _headers_default_handler(descriptor, spec, cls): + """Default handler when looking for the 'headers' attribute. + + Tries to search for ``*.h`` files recursively starting from + ``spec.prefix.include``. + + Parameters: + descriptor (ForwardQueryToPackage): descriptor that triggered the call + spec (Spec): spec that is being queried + cls (type(spec)): type of spec, to match the signature of the + descriptor ``__get__`` method + + Returns: + HeaderList: The headers in ``prefix.include`` + + Raises: + RuntimeError: If no headers are found + """ + headers = find_headers('*', root=spec.prefix.include, recurse=True) + + if headers: + return headers + else: + msg = 'Unable to locate {0} headers in {1}' + raise RuntimeError(msg.format(spec.name, spec.prefix.include)) + + +def _libs_default_handler(descriptor, spec, cls): + """Default handler when looking for the 'libs' attribute. + + Tries to search for ``lib{spec.name}`` recursively starting from + ``spec.prefix``. + + Parameters: + descriptor (ForwardQueryToPackage): descriptor that triggered the call + spec (Spec): spec that is being queried + cls (type(spec)): type of spec, to match the signature of the + descriptor ``__get__`` method + + Returns: + LibraryList: The libraries found + + Raises: + RuntimeError: If no libraries are found + """ + name = 'lib' + spec.name + + if '+shared' in spec: + libs = find_libraries( + name, root=spec.prefix, shared=True, recurse=True + ) + elif '~shared' in spec: + libs = find_libraries( + name, root=spec.prefix, shared=False, recurse=True + ) + else: + # Prefer shared + libs = find_libraries( + name, root=spec.prefix, shared=True, recurse=True + ) + if libs: + return libs + + libs = find_libraries( + name, root=spec.prefix, shared=False, recurse=True + ) + + if libs: + return libs + else: + msg = 'Unable to recursively locate {0} libraries in {1}' + raise RuntimeError(msg.format(spec.name, spec.prefix)) class ForwardQueryToPackage(object): @@ -797,7 +866,7 @@ class ForwardQueryToPackage(object): def __init__(self, attribute_name, default_handler=None): """Create a new descriptor. - Args: + Parameters: attribute_name (str): name of the attribute to be searched for in the Package instance default_handler (callable, optional): default function to be @@ -815,7 +884,7 @@ def __get__(self, instance, cls): """Retrieves the property from Package using a well defined chain of responsibility. - The order of call is : + The order of call is: 1. if the query was through the name of a virtual package try to search for the attribute `{virtual_name}_{attribute_name}` @@ -885,17 +954,21 @@ def __set__(self, instance, value): class SpecBuildInterface(ObjectWrapper): + command = ForwardQueryToPackage( + 'command', + default_handler=_command_default_handler + ) + + headers = ForwardQueryToPackage( + 'headers', + default_handler=_headers_default_handler + ) libs = ForwardQueryToPackage( 'libs', default_handler=_libs_default_handler ) - cppflags = ForwardQueryToPackage( - 'cppflags', - default_handler=_cppflags_default_handler - ) - def __init__(self, spec, name, query_parameters): super(SpecBuildInterface, self).__init__(spec) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index a4b35e1df70..6f8a2ef5d28 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -32,6 +32,7 @@ import pytest import spack import spack.store +from spack.util.executable import Executable from llnl.util.tty.colify import colify @@ -114,11 +115,17 @@ def test_default_queries(database): rec = install_db.get_record('zmpi') spec = rec.spec + libraries = spec['zmpi'].libs assert len(libraries) == 1 - cppflags_expected = '-I' + spec.prefix.include - assert spec['zmpi'].cppflags == cppflags_expected + headers = spec['zmpi'].headers + assert len(headers) == 1 + + command = spec['zmpi'].command + assert isinstance(command, Executable) + assert command.name == 'zmpi' + assert os.path.exists(command.path) def test_005_db_exists(database): diff --git a/lib/spack/spack/test/library_list.py b/lib/spack/spack/test/file_list.py similarity index 50% rename from lib/spack/spack/test/library_list.py rename to lib/spack/spack/test/file_list.py index 7fc2fd222fd..09517b4dabf 100644 --- a/lib/spack/spack/test/library_list.py +++ b/lib/spack/spack/test/file_list.py @@ -25,7 +25,7 @@ import unittest -from llnl.util.filesystem import LibraryList +from llnl.util.filesystem import LibraryList, HeaderList class LibraryListTest(unittest.TestCase): @@ -63,15 +63,29 @@ def test_flags(self): self.assertTrue('-L/dir2' in search_flags) self.assertTrue('-L/dir3' in search_flags) self.assertTrue(isinstance(search_flags, str)) + self.assertEqual( + search_flags, + '-L/dir1 -L/dir2 -L/dir3' + ) link_flags = self.liblist.link_flags + self.assertTrue('-llapack' in link_flags) + self.assertTrue('-lfoo' in link_flags) + self.assertTrue('-lblas' in link_flags) + self.assertTrue('-lbar' in link_flags) + self.assertTrue('-lbaz' in link_flags) + self.assertTrue(isinstance(link_flags, str)) self.assertEqual( link_flags, '-llapack -lfoo -lblas -lbar -lbaz' ) ld_flags = self.liblist.ld_flags - self.assertEqual(ld_flags, search_flags + ' ' + link_flags) + self.assertTrue(isinstance(ld_flags, str)) + self.assertEqual( + ld_flags, + search_flags + ' ' + link_flags + ) def test_paths_manipulation(self): names = self.liblist.names @@ -109,3 +123,103 @@ def test_add(self): type(pylist + self.liblist), type(self.liblist) ) + + +class HeaderListTest(unittest.TestCase): + def setUp(self): + h = [ + '/dir1/Python.h', + '/dir2/datetime.h', + '/dir1/pyconfig.h', + '/dir3/core.h', + 'pymem.h' + ] + headlist = HeaderList(h) + headlist.add_macro('-DBOOST_LIB_NAME=boost_regex') + headlist.add_macro('-DBOOST_DYN_LINK') + self.headlist = headlist + + def test_repr(self): + x = eval(repr(self.headlist)) + self.assertEqual(self.headlist, x) + + def test_joined_and_str(self): + s1 = self.headlist.joined() + self.assertEqual( + s1, + '/dir1/Python.h /dir2/datetime.h /dir1/pyconfig.h /dir3/core.h pymem.h' # NOQA: ignore=E501 + ) + s2 = str(self.headlist) + self.assertEqual(s1, s2) + s3 = self.headlist.joined(';') + self.assertEqual( + s3, + '/dir1/Python.h;/dir2/datetime.h;/dir1/pyconfig.h;/dir3/core.h;pymem.h' # NOQA: ignore=E501 + ) + + def test_flags(self): + include_flags = self.headlist.include_flags + self.assertTrue('-I/dir1' in include_flags) + self.assertTrue('-I/dir2' in include_flags) + self.assertTrue('-I/dir3' in include_flags) + self.assertTrue(isinstance(include_flags, str)) + self.assertEqual( + include_flags, + '-I/dir1 -I/dir2 -I/dir3' + ) + + macros = self.headlist.macro_definitions + self.assertTrue('-DBOOST_LIB_NAME=boost_regex' in macros) + self.assertTrue('-DBOOST_DYN_LINK' in macros) + self.assertTrue(isinstance(macros, str)) + self.assertEqual( + macros, + '-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK' + ) + + cpp_flags = self.headlist.cpp_flags + self.assertTrue(isinstance(cpp_flags, str)) + self.assertEqual( + cpp_flags, + include_flags + ' ' + macros + ) + + def test_paths_manipulation(self): + names = self.headlist.names + self.assertEqual( + names, + ['Python', 'datetime', 'pyconfig', 'core', 'pymem'] + ) + + directories = self.headlist.directories + self.assertEqual(directories, ['/dir1', '/dir2', '/dir3']) + + def test_get_item(self): + a = self.headlist[0] + self.assertEqual(a, '/dir1/Python.h') + + b = self.headlist[:] + self.assertEqual(type(b), type(self.headlist)) + self.assertEqual(self.headlist, b) + self.assertTrue(self.headlist is not b) + + def test_add(self): + pylist = [ + '/dir1/Python.h', # removed from the final list + '/dir2/pyconfig.h', + '/dir4/datetime.h' + ] + another = HeaderList(pylist) + h = self.headlist + another + self.assertEqual(len(h), 7) + # Invariant : l == l + l + self.assertEqual(h, h + h) + # Always produce an instance of HeaderList + self.assertEqual( + type(self.headlist), + type(self.headlist + pylist) + ) + self.assertEqual( + type(pylist + self.headlist), + type(self.headlist) + ) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 1d7f019fdf2..44865e7bdb0 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -53,8 +53,31 @@ def add_default_env(self, key, value): @property def command(self): + """The command-line string. + + Returns: + str: The executable and default arguments + """ return ' '.join(self.exe) + @property + def name(self): + """The executable name. + + Returns: + str: The basename of the executable + """ + return os.path.basename(self.path) + + @property + def path(self): + """The path to the executable. + + Returns: + str: The path to the executable + """ + return self.exe[0] + def __call__(self, *args, **kwargs): """Run this executable in a subprocess. diff --git a/var/spack/repos/builtin/packages/abinit/package.py b/var/spack/repos/builtin/packages/abinit/package.py index fe1c7ec7781..d73b89ea696 100644 --- a/var/spack/repos/builtin/packages/abinit/package.py +++ b/var/spack/repos/builtin/packages/abinit/package.py @@ -33,9 +33,9 @@ class Abinit(AutotoolsPackage): energy, charge density and electronic structure of systems made of electrons and nuclei (molecules and periodic solids) within Density Functional Theory (DFT), using pseudopotentials and a planewave - or wavelet basis. - - ABINIT also includes options to optimize the geometry according to the + or wavelet basis. + + ABINIT also includes options to optimize the geometry according to the DFT forces and stresses, or to perform molecular dynamics simulations using these forces, or to generate dynamical matrices, Born effective charges, and dielectric tensors, based on Density-Functional @@ -149,7 +149,7 @@ def configure_args(self): # LibXC library libxc = spec['libxc:fortran'] options.extend([ - 'with_libxc_incs={0}'.format(libxc.cppflags), + 'with_libxc_incs={0}'.format(libxc.headers.cpp_flags), 'with_libxc_libs={0}'.format(libxc.libs.ld_flags + ' -lm') ]) @@ -161,7 +161,7 @@ def configure_args(self): hdf5 = spec['hdf5:hl'] netcdff = spec['netcdf-fortran:shared'] options.extend([ - '--with-netcdf-incs={0}'.format(netcdff.cppflags), + '--with-netcdf-incs={0}'.format(netcdff.headers.cpp_flags), '--with-netcdf-libs={0}'.format( netcdff.libs.ld_flags + ' ' + hdf5.libs.ld_flags ), @@ -175,7 +175,7 @@ def configure_args(self): def check(self): """This method is called after the build phase if tests have been - explicitly activated by user. + explicitly activated by user. """ make('check') make('tests_in') diff --git a/var/spack/repos/builtin/packages/ack/package.py b/var/spack/repos/builtin/packages/ack/package.py index dd6685a829e..b377c30fe98 100644 --- a/var/spack/repos/builtin/packages/ack/package.py +++ b/var/spack/repos/builtin/packages/ack/package.py @@ -45,7 +45,7 @@ def install(self, spec, prefix): ack = 'ack-{0}-single-file'.format(self.version) # rewrite the script's #! line to call the perl dependency - shbang = '#!' + join_path(spec['perl'].prefix.bin, 'perl') + shbang = '#!' + spec['perl'].command.path filter_file(r'^#!/usr/bin/env perl', shbang, ack) install(ack, join_path(prefix.bin, "ack")) diff --git a/var/spack/repos/builtin/packages/blast-plus/package.py b/var/spack/repos/builtin/packages/blast-plus/package.py index 7cab51b0be9..3faf852739f 100644 --- a/var/spack/repos/builtin/packages/blast-plus/package.py +++ b/var/spack/repos/builtin/packages/blast-plus/package.py @@ -206,7 +206,7 @@ def configure_args(self): if '+python' in spec: config_args.append( - '--with-python={0}'.format(self.spec['python'].prefix) + '--with-python={0}'.format(self.spec['python'].home) ) else: config_args.append('--without-python') diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 487445ab102..d4c772ed037 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -25,7 +25,6 @@ from spack import * import sys import os -from glob import glob class Boost(Package): @@ -173,23 +172,11 @@ def determine_toolset(self, spec): return 'gcc' def bjam_python_line(self, spec): - from os.path import dirname, splitext - pydir = 'python%s.%s*' % spec['python'].version.version[:2] - incs = join_path(spec['python'].prefix.include, pydir, "pyconfig.h") - incs = glob(incs) - incs = " ".join([dirname(u) for u in incs]) - - pylib = 'libpython%s.%s*' % spec['python'].version.version[:2] - all_libs = join_path(spec['python'].prefix.lib, pylib) - libs = [u for u in all_libs if splitext(u)[1] == dso_suffix] - if len(libs) == 0: - libs = [u for u in all_libs if splitext(u)[1] == '.a'] - - libs = " ".join(libs) - return 'using python : %s : %s : %s : %s ;\n' % ( + return 'using python : {0} : {1} : {2} : {3} ;\n'.format( spec['python'].version.up_to(2), - join_path(spec['python'].prefix.bin, 'python'), - incs, libs + spec['python'].command.path, + spec['python'].headers.directories[0], + spec['python'].libs[0] ) def determine_bootstrap_options(self, spec, withLibs, options): @@ -198,7 +185,7 @@ def determine_bootstrap_options(self, spec, withLibs, options): options.append("--with-libraries=%s" % ','.join(withLibs)) if '+python' in spec: - options.append('--with-python=%s' % python_exe) + options.append('--with-python=%s' % spec['python'].command.path) with open('user-config.jam', 'w') as f: # Boost may end up using gcc even though clang+gfortran is set in diff --git a/var/spack/repos/builtin/packages/cantera/package.py b/var/spack/repos/builtin/packages/cantera/package.py index 0685772f82b..aa7c7dc98c7 100644 --- a/var/spack/repos/builtin/packages/cantera/package.py +++ b/var/spack/repos/builtin/packages/cantera/package.py @@ -117,15 +117,13 @@ def install(self, spec, prefix): if '+python' in spec: options.extend([ 'python_package=full', - 'python_cmd={0}'.format( - join_path(spec['python'].prefix.bin, 'python')), + 'python_cmd={0}'.format(spec['python'].command.path), 'python_array_home={0}'.format(spec['py-numpy'].prefix) ]) if spec['python'].satisfies('@3'): options.extend([ 'python3_package=y', - 'python3_cmd={0}'.format( - join_path(spec['python'].prefix.bin, 'python')), + 'python3_cmd={0}'.format(spec['python'].command.path), 'python3_array_home={0}'.format(spec['py-numpy'].prefix) ]) else: diff --git a/var/spack/repos/builtin/packages/conduit/package.py b/var/spack/repos/builtin/packages/conduit/package.py index ddefde4fb75..d97c6534e45 100644 --- a/var/spack/repos/builtin/packages/conduit/package.py +++ b/var/spack/repos/builtin/packages/conduit/package.py @@ -174,13 +174,13 @@ def create_host_config(self, spec, prefix): ############################################## if "+cmake" in spec: - cmake_exe = join_path(spec['cmake'].prefix.bin, "cmake") + cmake_exe = spec['cmake'].command.path else: cmake_exe = which("cmake") if cmake_exe is None: msg = 'failed to find CMake (and cmake variant is off)' raise RuntimeError(msg) - cmake_exe = cmake_exe.command + cmake_exe = cmake_exe.path host_cfg_fname = "%s-%s-%s.cmake" % (socket.gethostname(), sys_type, @@ -224,21 +224,15 @@ def create_host_config(self, spec, prefix): cfg.write("# Python Support\n") if "+python" in spec: - python_exe = join_path(spec['python'].prefix.bin, "python") cfg.write("# Enable python module builds\n") cfg.write(cmake_cache_entry("ENABLE_PYTHON", "ON")) cfg.write("# python from spack \n") - cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", python_exe)) + cfg.write(cmake_cache_entry("PYTHON_EXECUTABLE", + spec['python'].command.path)) # install module to standard style site packages dir # so we can support spack activate - py_ver_short = "python{0}".format(spec["python"].version.up_to(2)) - pym_prefix = join_path("${CMAKE_INSTALL_PREFIX}", - "lib", - py_ver_short, - "site-packages") - # use pym_prefix as the install path cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX", - pym_prefix)) + site_packages_dir)) else: cfg.write(cmake_cache_entry("ENABLE_PYTHON", "OFF")) @@ -251,7 +245,7 @@ def create_host_config(self, spec, prefix): cfg.write(cmake_cache_entry("SPHINX_EXECUTABLE", sphinx_build_exe)) cfg.write("# doxygen from uberenv\n") - doxygen_exe = join_path(spec['doxygen'].prefix.bin, "doxygen") + doxygen_exe = spec['doxygen'].command.path cfg.write(cmake_cache_entry("DOXYGEN_EXECUTABLE", doxygen_exe)) else: cfg.write(cmake_cache_entry("ENABLE_DOCS", "OFF")) diff --git a/var/spack/repos/builtin/packages/cosmomc/package.py b/var/spack/repos/builtin/packages/cosmomc/package.py index 1e83e02d656..6c41002079e 100644 --- a/var/spack/repos/builtin/packages/cosmomc/package.py +++ b/var/spack/repos/builtin/packages/cosmomc/package.py @@ -178,7 +178,7 @@ def check_install(self): os.environ.pop('CLIKPATH', '') os.environ.pop('PLANCKLIKE', '') - exe = join_path(prefix.bin, 'cosmomc') + exe = spec['cosmomc'].command.path args = [] if '+mpi' in spec: # Add mpirun prefix diff --git a/var/spack/repos/builtin/packages/cp2k/package.py b/var/spack/repos/builtin/packages/cp2k/package.py index 9bc1026ba9a..06e46979046 100644 --- a/var/spack/repos/builtin/packages/cp2k/package.py +++ b/var/spack/repos/builtin/packages/cp2k/package.py @@ -93,10 +93,10 @@ def install(self, spec, prefix): cppflags = [ '-D__FFTW3', '-D__LIBINT', - spec['fftw'].cppflags + spec['fftw'].headers.cpp_flags ] fcflags = copy.deepcopy(optflags[self.spec.compiler.name]) - fcflags.append(spec['fftw'].cppflags) + fcflags.append(spec['fftw'].headers.cpp_flags) fftw = find_libraries('libfftw3', root=spec['fftw'].prefix.lib) ldflags = [fftw.search_flags] if 'superlu-dist@4.3' in spec: @@ -161,14 +161,14 @@ def install(self, spec, prefix): cppflags.append('-D__WANNIER90') fcflags.extend([ - # spec['elpa:fortran'].cppflags + # spec['elpa:fortran'].headers.cpp_flags '-I' + join_path( spec['elpa'].prefix, 'include', 'elpa-{0}'.format(str(spec['elpa'].version)), 'modules' ), - # spec[pexsi:fortran].cppflags + # spec[pexsi:fortran].headers.cpp_flags '-I' + join_path(spec['pexsi'].prefix, 'fortran') ]) scalapack = spec['scalapack'].libs diff --git a/var/spack/repos/builtin/packages/foam-extend/package.py b/var/spack/repos/builtin/packages/foam-extend/package.py index 559cc45d7ab..c68e570c0fd 100644 --- a/var/spack/repos/builtin/packages/foam-extend/package.py +++ b/var/spack/repos/builtin/packages/foam-extend/package.py @@ -356,8 +356,8 @@ def configure(self, spec, prefix): 'CMAKE_BIN_DIR': spec['cmake'].prefix.bin, }, 'python': { - 'PYTHON_DIR': spec['python'].prefix, - 'PYTHON_BIN_DIR': spec['python'].prefix.bin, + 'PYTHON_DIR': spec['python'].home, + 'PYTHON_BIN_DIR': spec['python'].home.bin, }, 'flex': { 'FLEX_SYSTEM': 1, diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 00518c8cf48..b362060e612 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -176,7 +176,7 @@ def write_rpath_specs(self): self.spec.format('$_$@')) return - gcc = Executable(join_path(self.prefix.bin, 'gcc')) + gcc = self.spec['gcc'].command lines = gcc('-dumpspecs', output=str).strip().split("\n") specs_file = join_path(self.spec_dir, 'specs') with closing(open(specs_file, 'w')) as out: diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py index dc58cc89938..1b9a4f08182 100644 --- a/var/spack/repos/builtin/packages/gdal/package.py +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -72,7 +72,7 @@ def install(self, spec, prefix): args.append("--prefix=%s" % prefix) args.append("--with-liblzma=yes") args.append("--with-zlib=%s" % spec['zlib'].prefix) - args.append("--with-python=%s" % spec['python'].prefix.bin + "/python") + args.append("--with-python=%s" % spec['python'].command.path) args.append("--without-libtool") if '+geos' in spec: diff --git a/var/spack/repos/builtin/packages/geos/package.py b/var/spack/repos/builtin/packages/geos/package.py index 324186cfbc7..2419eed0389 100644 --- a/var/spack/repos/builtin/packages/geos/package.py +++ b/var/spack/repos/builtin/packages/geos/package.py @@ -60,9 +60,8 @@ class Geos(Package): def install(self, spec, prefix): args = ["--prefix=%s" % prefix] # if '+python' in spec: -# os.environ['PYTHON'] = join_path(spec['python'].prefix, 'bin', -# 'python' if spec['python'].version[:1][0] <= 2 else 'python3') -# os.environ['SWIG'] = join_path(spec['swig'].prefix, 'bin', 'swig') +# os.environ['PYTHON'] = spec['python'].command.path +# os.environ['SWIG'] = spec['swig'].command.path # # args.append("--enable-python") diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index dfa4e08643b..d71f49bff6a 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -162,8 +162,7 @@ def configure_args(self): '--with-iconv={0}'.format(spec['libiconv'].prefix), '--with-libpcre={0}'.format(spec['pcre'].prefix), '--with-openssl={0}'.format(spec['openssl'].prefix), - '--with-perl={0}'.format( - join_path(spec['perl'].prefix.bin, 'perl')), + '--with-perl={0}'.format(spec['perl'].command.path), '--with-zlib={0}'.format(spec['zlib'].prefix), ] diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 7f1523ca6d1..05a903ea749 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -117,7 +117,7 @@ def setup_dependent_package(self, module, dependent_spec): shutil.copytree('bin', os.path.join(prefix, '/bin')) """ # Add a go command/compiler for extensions - module.go = Executable(join_path(self.spec.prefix.bin, 'go')) + module.go = self.spec['go'].command def setup_dependent_environment(self, spack_env, run_env, dependent_spec): if os.environ.get('GOROOT', False): diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 79f23d7d92f..053a4d48524 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -69,7 +69,7 @@ def install(self, spec, prefix): # if sys.platform == "darwin": # fix_darwin_install_name(prefix.lib) - libtool = Executable(join_path(spec["libtool"].prefix.bin, "libtool")) + libtool = spec["libtool"].command # TODO: these vars are not used. # if "+mpi" in spec["hdf5"]: diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 4da24dd7f12..5a3f9e5657a 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -257,7 +257,7 @@ def check_install(self): cc = Executable(spec['mpi'].mpicc) else: cc = Executable(self.compiler.cc) - cc(*(['-c', "check.c"] + spec['hdf5'].cppflags.split())) + cc(*(['-c', "check.c"] + spec['hdf5'].headers.cpp_flags.split())) cc(*(['-o', "check", "check.o"] + spec['hdf5'].libs.ld_flags.split())) try: diff --git a/var/spack/repos/builtin/packages/hoomd-blue/package.py b/var/spack/repos/builtin/packages/hoomd-blue/package.py index 3d56f08dc0a..0aa9574a6d1 100644 --- a/var/spack/repos/builtin/packages/hoomd-blue/package.py +++ b/var/spack/repos/builtin/packages/hoomd-blue/package.py @@ -81,7 +81,7 @@ def cmake_args(self): spec = self.spec cmake_args = [ - '-DPYTHON_EXECUTABLE={0}/python'.format(spec['python'].prefix.bin), + '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path), ] # MPI support diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py index 4e7c65f53a4..007a384f10e 100644 --- a/var/spack/repos/builtin/packages/julia/package.py +++ b/var/spack/repos/builtin/packages/julia/package.py @@ -185,7 +185,7 @@ def install(self, spec, prefix): juliarc.write('\n') # Install some commonly used packages - julia = Executable(join_path(prefix.bin, "julia")) + julia = spec['julia'].command julia("-e", 'Pkg.init(); Pkg.update()') # Install HDF5 @@ -216,7 +216,7 @@ def install(self, spec, prefix): with open(join_path(prefix, "etc", "julia", "juliarc.jl"), "a") as juliarc: juliarc.write('# Python\n') - juliarc.write('ENV["PYTHON"] = "%s"\n' % spec["python"].prefix) + juliarc.write('ENV["PYTHON"] = "%s"\n' % spec["python"].home) juliarc.write('\n') # Python's OpenSSL package installer complains: # Error: PREFIX too long: 166 characters, but only 128 allowed diff --git a/var/spack/repos/builtin/packages/libxml2/package.py b/var/spack/repos/builtin/packages/libxml2/package.py index 470998d8826..4324b1dcd39 100644 --- a/var/spack/repos/builtin/packages/libxml2/package.py +++ b/var/spack/repos/builtin/packages/libxml2/package.py @@ -49,7 +49,7 @@ def configure_args(self): spec = self.spec if '+python' in spec: python_args = [ - '--with-python={0}'.format(spec['python'].prefix), + '--with-python={0}'.format(spec['python'].home), '--with-python-install-dir={0}'.format(site_packages_dir) ] else: diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index d2bf06e358a..868e3367787 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -339,7 +339,7 @@ def cmake_args(self): cmake_args = [ '-DLLVM_REQUIRES_RTTI:BOOL=ON', '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp', - '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix + '-DPYTHON_EXECUTABLE:PATH={0}'.format(spec['python'].command.path), ] if '+gold' in spec: diff --git a/var/spack/repos/builtin/packages/nwchem/package.py b/var/spack/repos/builtin/packages/nwchem/package.py index 3a8be3f56ea..10c5ec5701d 100644 --- a/var/spack/repos/builtin/packages/nwchem/package.py +++ b/var/spack/repos/builtin/packages/nwchem/package.py @@ -88,7 +88,7 @@ def install(self, spec, prefix): 'MPI_LOC=%s' % spec['mpi'].prefix, 'USE_PYTHONCONFIG=y', 'PYTHONVERSION=%s' % spec['python'].version.up_to(2), - 'PYTHONHOME=%s' % spec['python'].prefix, + 'PYTHONHOME=%s' % spec['python'].home, 'BLASOPT=%s' % ((lapack + blas).ld_flags), 'BLAS_LIB=%s' % blas.ld_flags, 'LAPACK_LIB=%s' % lapack.ld_flags, diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 81f48419317..4ebb38aa7aa 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -150,7 +150,7 @@ def check_install(self): blessed_file = join_path(os.path.dirname(self.module.__file__), 'test_cblas_dgemm.output') - include_flags = spec['openblas'].cppflags + include_flags = spec['openblas'].headers.cpp_flags link_flags = spec['openblas'].libs.ld_flags if self.compiler.name == 'intel': link_flags += ' -lifcore' diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py index 8a721032a61..b052cfa12ad 100644 --- a/var/spack/repos/builtin/packages/opencv/package.py +++ b/var/spack/repos/builtin/packages/opencv/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -from glob import glob class Opencv(Package): @@ -166,22 +165,11 @@ def install(self, spec, prefix): # Python if '+python' in spec: - python = spec['python'] - - try: - python_lib = glob(join_path( - python.prefix.lib, 'libpython*.{0}'.format(dso_suffix)))[0] - except KeyError: - raise InstallError('Cannot find libpython') - - try: - python_include_dir = glob(join_path(python.prefix.include, - 'python*'))[0] - except KeyError: - raise InstallError('Cannot find python include directory') + python_exe = spec['python'].command.path + python_lib = spec['python'].libs[0] + python_include_dir = spec['python'].headers.directories[0] if '^python@3:' in spec: - python_exe = join_path(python.prefix.bin, 'python3') cmake_options.extend([ '-DBUILD_opencv_python3=ON', '-DPYTHON3_EXECUTABLE={0}'.format(python_exe), @@ -190,7 +178,6 @@ def install(self, spec, prefix): '-DBUILD_opencv_python2=OFF', ]) elif '^python@2:3' in spec: - python_exe = join_path(python.prefix.bin, 'python2') cmake_options.extend([ '-DBUILD_opencv_python2=ON', '-DPYTHON2_EXECUTABLE={0}'.format(python_exe), diff --git a/var/spack/repos/builtin/packages/openspeedshop/package.py b/var/spack/repos/builtin/packages/openspeedshop/package.py index c4b150f56c4..6ad28ce3b76 100644 --- a/var/spack/repos/builtin/packages/openspeedshop/package.py +++ b/var/spack/repos/builtin/packages/openspeedshop/package.py @@ -167,24 +167,18 @@ def set_defaultbase_cmakeOptions(self, spec, cmakeOptions): # Appends to cmakeOptions the options that will enable # the appropriate base level options to the openspeedshop # cmake build. - python_vers = format(spec['python'].version.up_to(2)) - python_pv = '/python' + python_vers - python_pvs = '/libpython' + python_vers + '.' + format(dso_suffix) + python_exe = spec['python'].command.path + python_library = spec['python'].libs[0] + python_include = spec['python'].headers.directories[0] BaseOptions = [] BaseOptions.append('-DBINUTILS_DIR=%s' % spec['binutils'].prefix) BaseOptions.append('-DLIBELF_DIR=%s' % spec['libelf'].prefix) BaseOptions.append('-DLIBDWARF_DIR=%s' % spec['libdwarf'].prefix) - BaseOptions.append( - '-DPYTHON_EXECUTABLE=%s' - % join_path(spec['python'].prefix + '/bin/python')) - BaseOptions.append( - '-DPYTHON_INCLUDE_DIR=%s' - % join_path(spec['python'].prefix.include) + python_pv) - BaseOptions.append( - '-DPYTHON_LIBRARY=%s' - % join_path(spec['python'].prefix.lib) + python_pvs) + BaseOptions.append('-DPYTHON_EXECUTABLE=%s' % python_exe) + BaseOptions.append('-DPYTHON_INCLUDE_DIR=%s' % python_include) + BaseOptions.append('-DPYTHON_LIBRARY=%s' % python_library) BaseOptions.append('-DBoost_NO_SYSTEM_PATHS=TRUE') BaseOptions.append('-DBoost_NO_BOOST_CMAKE=TRUE') BaseOptions.append('-DBOOST_ROOT=%s' % spec['boost'].prefix) diff --git a/var/spack/repos/builtin/packages/pagmo/package.py b/var/spack/repos/builtin/packages/pagmo/package.py index 5a06b70e387..eeeb437db7f 100644 --- a/var/spack/repos/builtin/packages/pagmo/package.py +++ b/var/spack/repos/builtin/packages/pagmo/package.py @@ -110,7 +110,7 @@ def cmake_args(self): if '+python' in spec: args.extend([ # By default picks up the system python not the Spack build - '-DPYTHON_EXECUTABLE={0}'.format(python_exe), + '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path), # By default installs to the python prefix not the pagmo prefix '-DPYTHON_MODULES_DIR={0}'.format(site_packages_dir), ]) diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index 9b2b33625a0..ea0722dab8c 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -120,8 +120,7 @@ def nvariant_bool(feature): if '+python' in spec: cmake_args.extend([ '-DPARAVIEW_ENABLE_PYTHON:BOOL=ON', - '-DPYTHON_EXECUTABLE:FILEPATH=%s/bin/python' - % spec['python'].prefix + '-DPYTHON_EXECUTABLE:FILEPATH=%s' % spec['python'].command.path ]) if '+mpi' in spec: diff --git a/var/spack/repos/builtin/packages/perl/package.py b/var/spack/repos/builtin/packages/perl/package.py index ad67cae48c0..dc1f7be93cf 100644 --- a/var/spack/repos/builtin/packages/perl/package.py +++ b/var/spack/repos/builtin/packages/perl/package.py @@ -113,11 +113,10 @@ def install(self, spec, prefix): @run_after('install') def install_cpanm(self): spec = self.spec - prefix = self.prefix if '+cpanm' in spec: with working_dir(join_path('cpanm', 'cpanm')): - perl = Executable(join_path(prefix.bin, 'perl')) + perl = spec['perl'].command perl('Makefile.PL') make() make('install') diff --git a/var/spack/repos/builtin/packages/plumed/package.py b/var/spack/repos/builtin/packages/plumed/package.py index 60443cbcc65..3022cd7959e 100644 --- a/var/spack/repos/builtin/packages/plumed/package.py +++ b/var/spack/repos/builtin/packages/plumed/package.py @@ -122,7 +122,7 @@ def apply_patch(self, other): def setup_dependent_package(self, module, dependent_spec): # Make plumed visible from dependent packages - module.plumed = Executable(join_path(self.spec.prefix.bin, 'plumed')) + module.plumed = self.spec['plumed'].command @run_before('autoreconf') def filter_gslcblas(self): diff --git a/var/spack/repos/builtin/packages/pocl/package.py b/var/spack/repos/builtin/packages/pocl/package.py index b6baee7e072..8becd9e9f73 100644 --- a/var/spack/repos/builtin/packages/pocl/package.py +++ b/var/spack/repos/builtin/packages/pocl/package.py @@ -104,7 +104,7 @@ def check_install(self): with working_dir(checkdir, create=True): source = join_path(os.path.dirname(self.module.__file__), "example1.c") - cflags = spec["pocl"].cppflags.split() + cflags = spec["pocl"].headers.cpp_flags.split() # ldflags = spec["pocl"].libs.ld_flags.split() ldflags = ["-L%s" % spec["pocl"].prefix.lib, "-lOpenCL", "-lpoclu"] diff --git a/var/spack/repos/builtin/packages/psi4/package.py b/var/spack/repos/builtin/packages/psi4/package.py index d14aff363be..976b5d3c4ea 100644 --- a/var/spack/repos/builtin/packages/psi4/package.py +++ b/var/spack/repos/builtin/packages/psi4/package.py @@ -115,8 +115,7 @@ def filter_compilers(self, spec, prefix): ' -I'.join([ os.path.join(spec['psi4'].prefix.include, 'psi4'), os.path.join(spec['boost'].prefix.include, 'boost'), - os.path.join(spec['python'].prefix.include, 'python{0}'.format( - spec['python'].version.up_to(2))), + os.path.join(spec['python'].headers.directories[0]), spec['lapack'].prefix.include, spec['blas'].prefix.include, '/usr/include' diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index bc3de0479bb..12b77813d1c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -24,7 +24,9 @@ ############################################################################## import ast import os +import platform import re +import sys from contextlib import closing import spack @@ -33,10 +35,11 @@ from llnl.util.filesystem import force_remove from spack import * from spack.util.environment import * +from spack.util.prefix import Prefix import spack.util.spack_json as sjson -class Python(Package): +class Python(AutotoolsPackage): """The Python programming language.""" homepage = "http://www.python.org" @@ -62,6 +65,10 @@ class Python(Package): extendable = True + # --enable-shared is known to cause problems for some users on macOS + # See http://bugs.python.org/issue29846 + variant('shared', default=sys.platform != 'darwin', + description='Enable shared libraries') variant('tk', default=False, description='Provide support for Tkinter') variant('ucs4', default=False, description='Enable UCS4 (wide) unicode strings') @@ -82,13 +89,14 @@ class Python(Package): depends_on("tk", when="+tk") depends_on("tcl", when="+tk") - patch('ncurses.patch') + # Patch does not work for Python 3.1 + patch('ncurses.patch', when='@:2.8,3.2:') _DISTUTIL_VARS_TO_SAVE = ['LDSHARED'] _DISTUTIL_CACHE_FILENAME = 'sysconfig.json' _distutil_vars = None - @when('@2.7,3.4:') + @when('@2.7:2.8,3.4:') def patch(self): # NOTE: Python's default installation procedure makes it possible for a # user's local configurations to change the Spack installation. In @@ -100,7 +108,10 @@ def patch(self): r'\1setup.py\2 --no-user-cfg \3\6' ) - def install(self, spec, prefix): + def setup_environment(self, spack_env, run_env): + spec = self.spec + prefix = self.prefix + # TODO: The '--no-user-cfg' option for Python installation is only in # Python v2.7 and v3.4+ (see https://bugs.python.org/issue1180) and # adding support for ignoring user configuration will require @@ -110,23 +121,27 @@ def install(self, spec, prefix): 'user configurations are present.').format(self.version)) # Need this to allow python build to find the Python installation. - env['PYTHONHOME'], env['PYTHONPATH'] = prefix, prefix - env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' + spack_env.set('PYTHONHOME', prefix) + spack_env.set('PYTHONPATH', prefix) + spack_env.set('MACOSX_DEPLOYMENT_TARGET', platform.mac_ver()[0]) - # Rest of install is pretty standard except setup.py needs to - # be able to read the CPPFLAGS and LDFLAGS as it scans for the - # library and headers to build + def configure_args(self): + spec = self.spec + + # setup.py needs to be able to read the CPPFLAGS and LDFLAGS + # as it scans for the library and headers to build dep_pfxs = [dspec.prefix for dspec in spec.dependencies('link')] config_args = [ - '--prefix={0}'.format(prefix), '--with-threads', - '--enable-shared', 'CPPFLAGS=-I{0}'.format(' -I'.join(dp.include for dp in dep_pfxs)), 'LDFLAGS=-L{0}'.format(' -L'.join(dp.lib for dp in dep_pfxs)), ] - if spec.satisfies("platform=darwin") and ('%gcc' in spec): + if spec.satisfies('%gcc platform=darwin'): config_args.append('--disable-toolbox-glue') + if '+shared' in spec: + config_args.append('--enable-shared') + if '+ucs4' in spec: if spec.satisfies('@:2.7'): config_args.append('--enable-unicode=ucs4') @@ -140,18 +155,18 @@ def install(self, spec, prefix): if spec.satisfies('@3:'): config_args.append('--without-ensurepip') - configure(*config_args) - make() - make('install') + return config_args + + @run_after('install') + def post_install(self): + spec = self.spec + prefix = self.prefix self.sysconfigfilename = '_sysconfigdata.py' if spec.satisfies('@3.6:'): # Python 3.6.0 renamed the sys config file - python3 = os.path.join(prefix.bin, - 'python{0}'.format(self.version.up_to(1))) - python = Executable(python3) sc = 'import sysconfig; print(sysconfig._get_sysconfigdata_name())' - cf = python('-c', sc, output=str).strip('\n') + cf = self.command('-c', sc, output=str).strip() self.sysconfigfilename = '{0}.py'.format(cf) self._save_distutil_vars(prefix) @@ -205,7 +220,7 @@ def _save_distutil_vars(self, prefix): Spack partially covers this by setting environment variables that are also accounted for by distutils. Currently there is one more known variable that must be set, which is LDSHARED, so the method saves its - autogenerated value to pass it to the dependant package's setup script. + autogenerated value to pass it to the dependent package's setup script. """ self._distutil_vars = {} @@ -234,20 +249,20 @@ def _save_distutil_vars(self, prefix): pass if not input_dict: - tty.warn('Failed to find \'build_time_vars\' dictionary in file ' - '\'%s\'. This might cause the extensions that are ' - 'installed with distutils to call compilers directly ' - 'avoiding Spack\'s wrappers.' % input_filename) + tty.warn("Failed to find 'build_time_vars' dictionary in file " + "'%s'. This might cause the extensions that are " + "installed with distutils to call compilers directly " + "avoiding Spack's wrappers." % input_filename) return for var_name in Python._DISTUTIL_VARS_TO_SAVE: if var_name in input_dict: self._distutil_vars[var_name] = input_dict[var_name] else: - tty.warn('Failed to find key \'%s\' in \'build_time_vars\' ' - 'dictionary in file \'%s\'. This might cause the ' - 'extensions that are installed with distutils to ' - 'call compilers directly avoiding Spack\'s wrappers.' + tty.warn("Failed to find key '%s' in 'build_time_vars' " + "dictionary in file '%s'. This might cause the " + "extensions that are installed with distutils to " + "call compilers directly avoiding Spack's wrappers." % (var_name, input_filename)) if len(self._distutil_vars) > 0: @@ -259,10 +274,10 @@ def _save_distutil_vars(self, prefix): with open(output_filename, 'w') as output_file: sjson.dump(self._distutil_vars, output_file) except: - tty.warn('Failed to save metadata for distutils. This might ' - 'cause the extensions that are installed with ' - 'distutils to call compilers directly avoiding ' - 'Spack\'s wrappers.') + tty.warn("Failed to save metadata for distutils. This might " + "cause the extensions that are installed with " + "distutils to call compilers directly avoiding " + "Spack's wrappers.") # We make the cache empty if we failed to save it to file # to provide the same behaviour as in the case when the cache # is initialized by the method load_distutils_data(). @@ -319,6 +334,144 @@ def filter_compilers(self, prefix): # Set up environment to make install easy for python extensions. # ======================================================================== + @property + def command(self): + """Returns the Python command, which may vary depending + on the version of Python and how it was installed. + + In general, Python 2 comes with ``python`` and ``python2`` commands, + while Python 3 only comes with a ``python3`` command. + + :returns: The Python command + :rtype: Executable + """ + # We need to be careful here. If the user is using an externally + # installed python, all 3 commands could be in the same directory. + + # Search for `python2` iff using Python 2 + if (self.spec.satisfies('@:2') and + os.path.exists(os.path.join(self.prefix.bin, 'python2'))): + command = 'python2' + # Search for `python3` iff using Python 3 + elif (self.spec.satisfies('@3:') and + os.path.exists(os.path.join(self.prefix.bin, 'python3'))): + command = 'python3' + # If neither were found, try `python` + elif os.path.exists(os.path.join(self.prefix.bin, 'python')): + command = 'python' + else: + msg = 'Unable to locate {0} command in {1}' + raise RuntimeError(msg.format(self.name, self.prefix.bin)) + + # The python command may be a symlink if it was installed + # with Homebrew. Since some packages try to determine the + # location of libraries and headers based on the path, + # return the realpath + path = os.path.realpath(os.path.join(self.prefix.bin, command)) + + return Executable(path) + + def print_string(self, string): + """Returns the appropriate print string depending on the + version of Python. + + Examples: + + * Python 2 + + .. code-block:: python + + >>> self.print_string('sys.prefix') + 'print sys.prefix' + + * Python 3 + + .. code-block:: python + + >>> self.print_string('sys.prefix') + 'print(sys.prefix)' + """ + if self.spec.satisfies('@:2'): + return 'print {0}'.format(string) + else: + return 'print({0})'.format(string) + + def get_config_var(self, key): + """Returns the value of a single variable. Wrapper around + ``distutils.sysconfig.get_config_var()``.""" + + cmd = 'from distutils.sysconfig import get_config_var; ' + cmd += self.print_string("get_config_var('{0}')".format(key)) + + return self.command('-c', cmd, output=str).strip() + + def get_config_h_filename(self): + """Returns the full path name of the configuration header. + Wrapper around ``distutils.sysconfig.get_config_h_filename()``.""" + + cmd = 'from distutils.sysconfig import get_config_h_filename; ' + cmd += self.print_string('get_config_h_filename()') + + return self.command('-c', cmd, output=str).strip() + + @property + def home(self): + """Most of the time, ``PYTHONHOME`` is simply + ``spec['python'].prefix``. However, if the user is using an + externally installed python, it may be symlinked. For example, + Homebrew installs python in ``/usr/local/Cellar/python/2.7.12_2`` + and symlinks it to ``/usr/local``. Users may not know the actual + installation directory and add ``/usr/local`` to their + ``packages.yaml`` unknowingly. Query the python executable to + determine exactly where it is installed.""" + + prefix = self.get_config_var('prefix') + return Prefix(prefix) + + @property + def libs(self): + # Spack installs libraries into lib, except on openSUSE where it + # installs them into lib64. If the user is using an externally + # installed package, it may be in either lib or lib64, so we need + # to ask Python where its LIBDIR is. + libdir = self.get_config_var('LIBDIR') + + # The system Python installation on macOS and Homebrew installations + # install libraries into a Frameworks directory + frameworkprefix = self.get_config_var('PYTHONFRAMEWORKPREFIX') + + if '+shared' in self.spec: + ldlibrary = self.get_config_var('LDLIBRARY') + + if os.path.exists(os.path.join(libdir, ldlibrary)): + return LibraryList(os.path.join(libdir, ldlibrary)) + elif os.path.exists(os.path.join(frameworkprefix, ldlibrary)): + return LibraryList(os.path.join(frameworkprefix, ldlibrary)) + else: + msg = 'Unable to locate {0} libraries in {1}' + raise RuntimeError(msg.format(self.name, libdir)) + else: + library = self.get_config_var('LIBRARY') + + if os.path.exists(os.path.join(libdir, library)): + return LibraryList(os.path.join(libdir, library)) + elif os.path.exists(os.path.join(frameworkprefix, library)): + return LibraryList(os.path.join(frameworkprefix, library)) + else: + msg = 'Unable to locate {0} libraries in {1}' + raise RuntimeError(msg.format(self.name, libdir)) + + @property + def headers(self): + config_h = self.get_config_h_filename() + + if os.path.exists(config_h): + return HeaderList(config_h) + else: + includepy = self.get_config_var('INCLUDEPY') + msg = 'Unable to locate {0} headers in {1}' + raise RuntimeError(msg.format(self.name, includepy)) + @property def python_lib_dir(self): return join_path('lib', 'python{0}'.format(self.version.up_to(2))) @@ -332,29 +485,10 @@ def site_packages_dir(self): return join_path(self.python_lib_dir, 'site-packages') def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - """Set PYTHONPATH to include site-packages dir for the + """Set PYTHONPATH to include the site-packages directory for the extension and any other python extensions it depends on.""" - # The python executable for version 3 may be python3 or python - # See https://github.com/LLNL/spack/pull/2173#issuecomment-257170199 - pythonex = 'python{0}'.format('3' if self.spec.satisfies('@3') else '') - if os.path.isdir(self.prefix.bin): - base = self.prefix.bin - else: - base = self.prefix - if not os.path.isfile(os.path.join(base, pythonex)): - if self.spec.satisfies('@3'): - python = Executable(os.path.join(base, 'python')) - version = python('-c', 'import sys; print(sys.version)', - output=str) - if version.startswith('3'): - pythonex = 'python' - else: - raise RuntimeError('Cannot locate python executable') - else: - raise RuntimeError('Cannot locate python executable') - python = Executable(os.path.join(base, pythonex)) - prefix = python('-c', 'import sys; print(sys.prefix)', output=str) - spack_env.set('PYTHONHOME', prefix.strip('\n')) + + spack_env.set('PYTHONHOME', self.home) python_paths = [] for d in dependent_spec.traverse( @@ -378,19 +512,15 @@ def setup_dependent_package(self, module, dependent_spec): In most cases, extensions will only need to have one line:: setup_py('install', '--prefix={0}'.format(prefix))""" - python_path = join_path( - self.spec.prefix.bin, - 'python{0}'.format('3' if self.spec.satisfies('@3') else '') - ) - module.python_exe = python_path - module.python = Executable(python_path) - module.setup_py = Executable(python_path + ' setup.py --no-user-cfg') + module.python = self.command + module.setup_py = Executable( + self.command.path + ' setup.py --no-user-cfg') distutil_vars = self._load_distutil_vars() if distutil_vars: - for key, value in distutil_vars.iteritems(): + for key, value in distutil_vars.items(): module.setup_py.add_default_env(key, value) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. @@ -429,7 +559,7 @@ def python_ignore(self, ext_pkg, args): if ext_pkg.name != 'py-pygments': patterns.append(r'bin/pygmentize$') if ext_pkg.name != 'py-numpy': - patterns.append(r'bin/f2py3?$') + patterns.append(r'bin/f2py[0-9.]*$') return match_predicate(ignore_arg, patterns) @@ -457,7 +587,7 @@ def write_easy_install_pth(self, exts): paths.append(line) - site_packages = join_path(self.prefix, self.site_packages_dir) + site_packages = join_path(self.home, self.site_packages_dir) main_pth = join_path(site_packages, "easy-install.pth") if not paths: diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py index 8efb6294871..89d1f8e568f 100644 --- a/var/spack/repos/builtin/packages/scotch/package.py +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -22,7 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os from spack import * @@ -165,8 +164,8 @@ def configure(self): # General Features # - flex_path = os.path.join(self.spec['flex'].prefix.bin, 'flex') - bison_path = os.path.join(self.spec['bison'].prefix.bin, 'bison') + flex_path = self.spec['flex'].command.path + bison_path = self.spec['bison'].command.path makefile_inc.extend([ 'EXE =', 'OBJ = .o', diff --git a/var/spack/repos/builtin/packages/shiny-server/package.py b/var/spack/repos/builtin/packages/shiny-server/package.py index e8a899115f3..a3a2b2b1600 100644 --- a/var/spack/repos/builtin/packages/shiny-server/package.py +++ b/var/spack/repos/builtin/packages/shiny-server/package.py @@ -54,9 +54,7 @@ def cmake_args(self): spec = self.spec options = [] - options.extend([ - "-DPYTHON=%s" % join_path(spec['python'].prefix.bin, 'python'), - ]) + options.append("-DPYTHON=%s" % spec['python'].command.path) return options diff --git a/var/spack/repos/builtin/packages/spark/package.py b/var/spack/repos/builtin/packages/spark/package.py index 24dca5571df..3bc6abd234c 100644 --- a/var/spack/repos/builtin/packages/spark/package.py +++ b/var/spack/repos/builtin/packages/spark/package.py @@ -69,8 +69,7 @@ def setup_environment(self, spack_env, run_env): env['JAVA_HOME'] = self.spec['jdk'].prefix # spack_env.set('JAVA_HOME', self.spec['jdk'].prefix) - hadoop_bin_path = join_path(self.spec['hadoop'].prefix.bin, 'hadoop') - hadoop_bin = Executable(hadoop_bin_path) - hadoop_classpath = hadoop_bin('classpath', return_output=True) + hadoop = self.spec['hadoop'].command + hadoop_classpath = hadoop('classpath', return_output=True) run_env.set('SPARK_DIST_CLASSPATH', hadoop_classpath) diff --git a/var/spack/repos/builtin/packages/stat/package.py b/var/spack/repos/builtin/packages/stat/package.py index 6fe82baa2f3..59d110330a9 100644 --- a/var/spack/repos/builtin/packages/stat/package.py +++ b/var/spack/repos/builtin/packages/stat/package.py @@ -69,7 +69,7 @@ def configure_args(self): "--with-graphlib=%s" % spec['graphlib'].prefix, "--with-stackwalker=%s" % spec['dyninst'].prefix, "--with-libdwarf=%s" % spec['libdwarf'].prefix, - "--with-python=%s/bin/python" % spec['python'].prefix + "--with-python=%s" % spec['python'].command.path, ] if '+dysect' in spec: args.append('--enable-dysectapi') diff --git a/var/spack/repos/builtin/packages/subversion/package.py b/var/spack/repos/builtin/packages/subversion/package.py index c7057a51df0..628dd26d246 100644 --- a/var/spack/repos/builtin/packages/subversion/package.py +++ b/var/spack/repos/builtin/packages/subversion/package.py @@ -72,8 +72,7 @@ def install(self, spec, prefix): if 'swig' in spec: options.append('--with-swig=%s' % spec['swig'].prefix) if 'perl' in spec: - options.append( - 'PERL=%s' % join_path(spec['perl'].prefix.bin, 'perl')) + options.append('PERL=%s' % spec['perl'].command.path) configure(*options) make() diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py index e19a33a9000..e7d7b1587d6 100644 --- a/var/spack/repos/builtin/packages/superlu-dist/package.py +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -72,8 +72,8 @@ def install(self, spec, prefix): 'RANLIB = true', 'CC = {0}'.format(self.spec['mpi'].mpicc), 'CFLAGS = -fPIC -std=c99 -O2 %s %s %s' % ( - spec['parmetis'].cppflags, - spec['metis'].cppflags, + spec['parmetis'].headers.cpp_flags, + spec['metis'].headers.cpp_flags, '-D_LONGINT' if '+int64' in spec else ''), 'NOOPTS = -fPIC -std=c99', 'FORTRAN = {0}'.format(self.spec['mpi'].mpif77), diff --git a/var/spack/repos/builtin/packages/visit/package.py b/var/spack/repos/builtin/packages/visit/package.py index 48a3762a2a7..3c8505db972 100644 --- a/var/spack/repos/builtin/packages/visit/package.py +++ b/var/spack/repos/builtin/packages/visit/package.py @@ -52,7 +52,7 @@ def install(self, spec, prefix): '-DVTK_MINOR_VERSION=1', '-DVISIT_USE_GLEW=OFF', '-DVISIT_LOC_QMAKE_EXE:FILEPATH={0}/qmake-qt4'.format(qt_bin), - '-DPYTHON_DIR:PATH={0}'.format(spec['python'].prefix), + '-DPYTHON_DIR:PATH={0}'.format(spec['python'].home), '-DVISIT_SILO_DIR:PATH={0}'.format(spec['silo'].prefix), '-DVISIT_HDF5_DIR:PATH={0}'.format(spec['hdf5'].prefix), '-DVISIT_VTK_DIR:PATH={0}'.format(spec['vtk'].prefix),