zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added feature: package extensions

Browse Source 
- packages can be "extended" by others
- allows extension to be symlinked into extendee's prefix.
- used for python modules.
  - first module: py-setuptools
This commit is contained in:
Todd Gamblin
2015-01-07 11:48:21 -08:00
parent 7215aee224
commit 9977543478
7 changed files with 261 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
lib/spack/llnl/util/filesystem.py
View File

@@ -24,7 +24,8 @@
##############################################################################
__all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir',
'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor',
'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe']
'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe',
'check_link_tree', 'merge_link_tree', 'unmerge_link_tree']
import os
import sys
@@ -222,3 +223,82 @@ def ancestor(dir, n=1):
def can_access(file_name):
"""True if we have read/write access to the file."""
return os.access(file_name, os.R_OK|os.W_OK)
def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs):
# Yield directories before or after their contents.
order = kwargs.get('order', 'pre')
if order not in ('pre', 'post'):
raise ValueError("Order must be 'pre' or 'post'.")
# List of relative paths to ignore under the src root.
ignore = kwargs.get('ignore', None)
if isinstance(ignore, basestring):
ignore = (ignore,)
for dirpath, dirnames, filenames in os.walk(src_root):
rel_path = dirpath[len(src_root):]
rel_path = rel_path.lstrip(os.path.sep)
dest_dirpath = os.path.join(dest_root, rel_path)
# Don't descend into ignored directories
if ignore and dest_dirpath in ignore:
return
# Don't descend into dirs in dest that do not exist in src.
if not follow_nonexisting:
dirnames[:] = [
d for d in dirnames
if os.path.exists(os.path.join(dest_dirpath, d))]
# preorder yields directories before children
if order == 'pre':
yield (dirpath, dest_dirpath)
for name in filenames:
src_file = os.path.join(dirpath, name)
dest_file = os.path.join(dest_dirpath, name)
# Ignore particular paths inside the install root.
src_relpath = src_file[len(src_root):]
src_relpath = src_relpath.lstrip(os.path.sep)
if ignore and src_relpath in ignore:
continue
yield (src_file, dest_file)
# postorder yields directories after children
if order == 'post':
yield (dirpath, dest_dirpath)
def check_link_tree(src_root, dest_root, **kwargs):
for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs):
if os.path.exists(dest) and not os.path.isdir(dest):
return dest
return None
def merge_link_tree(src_root, dest_root, **kwargs):
kwargs['order'] = 'pre'
for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
if os.path.isdir(src):
mkdirp(dest)
else:
assert(not os.path.exists(dest))
os.symlink(src, dest)
def unmerge_link_tree(src_root, dest_root, **kwargs):
kwargs['order'] = 'post'
for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
if os.path.isdir(dest):
if not os.listdir(dest):
# TODO: what if empty directories were present pre-merge?
shutil.rmtree(dest, ignore_errors=True)
elif os.path.exists(dest):
if not os.path.islink(dest):
raise ValueError("%s is not a link tree!" % dest)
os.remove(dest)

2
lib/spack/spack/__init__.py
View File

@@ -138,7 +138,7 @@
# should live. This file is overloaded for spack core vs. for packages.
#
__all__ = ['Package', 'Version', 'when', 'ver']
from spack.package import Package
from spack.package import Package, ExtensionConflictError
from spack.version import Version, ver
from spack.multimethod import when

55
lib/spack/spack/directory_layout.py
View File

@@ -53,6 +53,19 @@ def __init__(self, root):
self.root = root
@property
def hidden_file_paths(self):
"""Return a list of hidden files used by the directory layout.
Paths are relative to the root of an install directory.
If the directory layout uses no hidden files to maintain
state, this should return an empty container, e.g. [] or (,).
"""
raise NotImplementedError()
def all_specs(self):
"""To be implemented by subclasses to traverse all specs for which there is
a directory within the root.
@@ -156,6 +169,11 @@ def __init__(self, root, **kwargs):
self.extension_file_name = extension_file_name
@property
def hidden_file_paths(self):
return ('.spec', '.extensions')
def relative_path_for_spec(self, spec):
_check_concrete(spec)
dir_name = spec.format('$_$@$+$#')
@@ -249,28 +267,32 @@ def extension_file_path(self, spec):
def get_extensions(self, spec):
path = self.extension_file_path(spec)
_check_concrete(spec)
path = self.extension_file_path(spec)
extensions = set()
if os.path.exists(path):
with closing(open(path)) as spec_file:
for line in spec_file:
with closing(open(path)) as ext_file:
for line in ext_file:
try:
extensions.add(Spec(line))
except SpecError, e:
extensions.add(Spec(line.strip()))
except spack.error.SpackError, e:
raise InvalidExtensionSpecError(str(e))
return extensions
def write_extensions(self, extensions):
def write_extensions(self, spec, extensions):
path = self.extension_file_path(spec)
with closing(open(path, 'w')) as spec_file:
for extension in sorted(extensions):
spec_file.write("%s\n" % extensions)
spec_file.write("%s\n" % extension)
def add_extension(self, spec, extension_spec):
exts = get_extensions(spec)
_check_concrete(spec)
_check_concrete(extension_spec)
exts = self.get_extensions(spec)
if extension_spec in exts:
raise ExtensionAlreadyInstalledError(spec, extension_spec)
else:
@@ -279,16 +301,19 @@ def add_extension(self, spec, extension_spec):
raise ExtensionConflictError(spec, extension_spec, already_installed)
exts.add(extension_spec)
self.write_extensions(exts)
self.write_extensions(spec, exts)
def remove_extension(self, spec, extension_spec):
exts = get_extensions(spec)
_check_concrete(spec)
_check_concrete(extension_spec)
exts = self.get_extensions(spec)
if not extension_spec in exts:
raise NoSuchExtensionError(spec, extension_spec)
exts.remove(extension_spec)
self.write_extensions(exts)
self.write_extensions(spec, exts)
class DirectoryLayoutError(SpackError):
@@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError):
"""Raised when an extension is added to a package that already has it."""
def __init__(self, spec, extension_spec):
super(ExtensionAlreadyInstalledError, self).__init__(
"%s is already installed in %s" % (extension_spec, spec))
"%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec))
class ExtensionConflictError(DirectoryLayoutError):
@@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError):
def __init__(self, spec, extension_spec, conflict):
super(ExtensionConflictError, self).__init__(
"%s cannot be installed in %s because it conflicts with %s."% (
extension_spec, spec, conflict))
extension_spec.short_spec, spec.short_spec, conflict.short_spec))
class NoSuchExtensionError(DirectoryLayoutError):
"""Raised when an extension isn't there on remove."""
def __init__(self, spec, extension_spec):
super(NoSuchExtensionError, self).__init__(
"%s cannot be removed from %s beacuse it's not installed."% (
extension_spec, spec, conflict))
"%s cannot be removed from %s because it's not installed."% (
extension_spec.short_spec, spec.short_spec))

49
lib/spack/spack/hooks/extensions.py Normal file
View File

@@ -0,0 +1,49 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import spack
def post_install(pkg):
assert(pkg.spec.concrete)
for name, spec in pkg.extendees.items():
ext = pkg.spec[name]
epkg = ext.package
if epkg.installed:
epkg.do_activate(pkg)
def pre_uninstall(pkg):
assert(pkg.spec.concrete)
# Need to do this b/c uninstall does not automatically do it.
# TODO: store full graph info in stored .spec file.
pkg.spec.normalize()
for name, spec in pkg.extendees.items():
ext = pkg.spec[name]
epkg = ext.package
if epkg.installed:
epkg.do_deactivate(pkg)

85
lib/spack/spack/package.py
View File

@@ -329,6 +329,9 @@ class SomePackage(Package):
"""By default we build in parallel. Subclasses can override this."""
parallel = True
"""Most packages are NOT extendable. Set to True if you want extensions."""
extendable = False
def __init__(self, spec):
# this determines how the package should be built.
@@ -398,6 +401,9 @@ def ensure_has_dict(attr_name):
self._fetch_time = 0.0
self._total_time = 0.0
for name, spec in self.extendees.items():
spack.db.get(spec)._check_extendable()
@property
def version(self):
@@ -877,6 +883,79 @@ def do_uninstall(self, **kwargs):
spack.hooks.post_uninstall(self)
def _check_extendable(self):
if not self.extendable:
raise ValueError("Package %s is not extendable!" % self.name)
def _sanity_check_extension(self, extension):
self._check_extendable()
if not self.installed:
raise ValueError("Can only (de)activate extensions for installed packages.")
if not extension.installed:
raise ValueError("Extensions must first be installed.")
if not self.name in extension.extendees:
raise ValueError("%s does not extend %s!" % (extension.name, self.name))
if not self.spec.satisfies(extension.extendees[self.name]):
raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec))
def do_activate(self, extension):
self._sanity_check_extension(extension)
self.activate(extension)
spack.install_layout.add_extension(self.spec, extension.spec)
tty.msg("Activated extension %s for %s."
% (extension.spec.short_spec, self.spec.short_spec))
def activate(self, extension):
"""Symlinks all files from the extension into extendee's install dir.
Package authors can override this method to support other
extension mechanisms. Spack internals (commands, hooks, etc.)
should call do_activate() method so that proper checks are
always executed.
"""
conflict = check_link_tree(
extension.prefix, self.prefix,
ignore=spack.install_layout.hidden_file_paths)
if conflict:
raise ExtensionConflictError(conflict)
merge_link_tree(extension.prefix, self.prefix,
ignore=spack.install_layout.hidden_file_paths)
def do_deactivate(self, extension):
self._sanity_check_extension(extension)
self.deactivate(extension)
ext = extension.spec
if ext in spack.install_layout.get_extensions(self.spec):
spack.install_layout.remove_extension(self.spec, ext)
tty.msg("Deactivated extension %s for %s."
% (extension.spec.short_spec, self.spec.short_spec))
def deactivate(self, extension):
"""Unlinks all files from extension out of extendee's install dir.
Package authors can override this method to support other
extension mechanisms. Spack internals (commands, hooks, etc.)
should call do_deactivate() method so that proper checks are
always executed.
"""
unmerge_link_tree(extension.prefix, self.prefix,
ignore=spack.install_layout.hidden_file_paths)
tty.msg("Deactivated %s as extension of %s."
% (extension.spec.short_spec, self.spec.short_spec))
def do_clean(self):
if self.stage.expanded_archive_path:
self.stage.chdir_to_source()
@@ -1068,3 +1147,9 @@ class NoURLError(PackageError):
def __init__(self, cls):
super(NoURLError, self).__init__(
"Package %s has no version with a URL." % cls.__name__)
class ExtensionConflictError(PackageError):
def __init__(self, path):
super(ExtensionConflictError, self).__init__(
"Extension blocked by file: %s" % path)

4
lib/spack/spack/relations.py
View File

@@ -68,7 +68,7 @@ class Mpileaks(Package):
spack install mpileaks ^mvapich
spack install mpileaks ^mpich
"""
__all__ = [ 'depends_on', 'provides', 'patch', 'version' ]
__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ]
import re
import inspect
@@ -135,7 +135,7 @@ def extends(*specs):
for string in specs:
for spec in spack.spec.parse(string):
if pkg == spec.name:
raise CircularReferenceError('depends_on', pkg)
raise CircularReferenceError('extends', pkg)
dependencies[spec.name] = spec
extendees[spec.name] = spec

3
var/spack/packages/python/package.py
View File

@@ -1,10 +1,13 @@
from spack import *
class Python(Package):
"""The Python programming language."""
homepage = "http://www.python.org"
url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz"
extendable = True
version('2.7.8', 'd235bdfa75b8396942e360a70487ee00')
depends_on("openssl")