zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

views: packages can customize how they're added to views (#7152)

Browse Source 
Functional updates:

- `python` now creates a copy of the `python` binaries when it is added
  to a view

- Python extensions (packages which subclass `PythonPackage`) rewrite
  their shebang lines to refer to python in the view

- Python packages in the same namespace will not generate conflicts if
  both have `...lib/site-packages/namespace-example/__init__.py`

  - These `__init__` files will also remain when removing any package in
    the namespace until the last package in the namespace is removed


Generally (Updated 2/16):

- Any package can define `add_files_to_view` to customize how it is added
  to a view (and at the moment custom definitions are included for
  `python` and `PythonPackage`)

  - Likewise any package can define `remove_files_from_view` to customize
    which files are removed (e.g. you don't always want to remove the
    namespace `__init__`)

- Any package can define `view_file_conflicts` to customize what it
  considers a merge conflict

- Global activations are handled like views (where the view root is the
  spec prefix of the extendee)

  - Benefit: filesystem-management aspects of activating extensions are
    now placed in views (e.g. now one can hardlink a global activation)

  - Benefit: overriding `Package.activate` is more straightforward (see
    `Python.activate`)

  - Complication: extension packages which have special-purpose logic
    *only* when activated outside of the extendee prefix must check for
    this in their `add_files_to_view` method (see `PythonPackage`)

- `LinkTree` is refactored to have separate methods for copying a
  directory structure and for copying files (since it was found that
  generally packages may want to alter how files are copied but still
  wanted to copy directories in the same way)


TODOs (updated 2/20):

- [x] additional testing (there is some unit testing added at this point
  but more would be useful)

- [x] refactor or reorganize `LinkTree` methods: currently there is a
  separate set of methods for replicating just the directory structure
  without the files, and a set for replicating everything

- [x] Right now external views (i.e. those not used for global
  activations) call `view.add_extension`, but global activations do not
  to avoid some extra work that goes into maintaining external views. I'm
  not sure if addressing that needs to be done here but I'd like to
  clarify it in the comments (UPDATE: for now I have added a TODO and in
  my opinion this can be merged now and the refactor handled later)

- [x] Several method descriptions (e.g. for `Package.activate`) are out
  of date and reference a distinction between global activations and
  views, they need to be updated

- [x] Update aspell package activations
This commit is contained in:
scheibelp
2018-06-26 16:14:05 -07:00
committed by Todd Gamblin
parent 1276ce0585
commit 3560f6dbe9
26 changed files with 837 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
var/spack/repos/builtin/packages/aspell/package.py
View File

@@ -23,9 +23,6 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
from llnl.util.link_tree import LinkTree
import spack.store
from spack.package import ExtensionError, ExtensionConflictError
# See also: AspellDictPackage
@@ -41,47 +38,3 @@ class Aspell(AutotoolsPackage):
version('0.60.6.1', 'e66a9c9af6a60dc46134fdacf6ce97d7')
patch('darwin.patch', when='platform=darwin')
# The dictionaries install all their bits into their prefix.lib dir,
# we want to link them into aspell's dict-dir.
# These are identical to what's in spack/package.py except
# for using:
# - extension.prefix.lib instead of extension.prefix in LinkTree()
# - dest_dir instead of self.prefix in tree.(find_conflict|merge)()
def activate(self, extension, **kwargs):
extensions_layout = kwargs.get("extensions_layout",
spack.store.extensions)
if extensions_layout is not spack.store.extensions:
raise ExtensionError(
'aspell does not support non-global extensions')
aspell = which(self.prefix.bin.aspell)
dest_dir = aspell('dump', 'config', 'dict-dir', output=str).strip()
tree = LinkTree(extension.prefix.lib)
def ignore(filename):
return (filename in spack.store.layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename))
conflict = tree.find_conflict(dest_dir, ignore=ignore)
if conflict:
raise ExtensionConflictError(conflict)
tree.merge(dest_dir, ignore=ignore)
def deactivate(self, extension, **kwargs):
extensions_layout = kwargs.get("extensions_layout",
spack.store.extensions)
if extensions_layout is not spack.store.extensions:
raise ExtensionError(
'aspell does not support non-global extensions')
aspell = which(self.prefix.bin.aspell)
dest_dir = aspell('dump', 'config', 'dict-dir', output=str).strip()
def ignore(filename):
return (filename in spack.store.layout.hidden_file_paths or
kwargs.get('ignore', lambda f: False)(filename))
tree = LinkTree(extension.prefix.lib)
tree.unmerge(dest_dir, ignore=ignore)

17
var/spack/repos/builtin/packages/perl/package.py
View File

@@ -35,7 +35,6 @@
from llnl.util.lang import match_predicate
import spack.store
from spack import *
@@ -267,27 +266,23 @@ def perl_ignore(self, ext_pkg, args):
return match_predicate(ignore_arg, patterns)
def activate(self, ext_pkg, **args):
def activate(self, ext_pkg, view, **args):
ignore = self.perl_ignore(ext_pkg, args)
args.update(ignore=ignore)
super(Perl, self).activate(ext_pkg, **args)
extensions_layout = args.get("extensions_layout",
spack.store.extensions)
super(Perl, self).activate(ext_pkg, view, **args)
extensions_layout = view.extensions_layout
exts = extensions_layout.extension_map(self.spec)
exts[ext_pkg.name] = ext_pkg.spec
def deactivate(self, ext_pkg, **args):
def deactivate(self, ext_pkg, view, **args):
ignore = self.perl_ignore(ext_pkg, args)
args.update(ignore=ignore)
super(Perl, self).deactivate(ext_pkg, **args)
extensions_layout = args.get("extensions_layout",
spack.store.extensions)
super(Perl, self).deactivate(ext_pkg, view, **args)
extensions_layout = view.extensions_layout
exts = extensions_layout.extension_map(self.spec)
# Make deactivate idempotent
if ext_pkg.name in exts:

2
var/spack/repos/builtin/packages/py-backports-shutil-get-terminal-size/package.py
View File

@@ -32,6 +32,8 @@ class PyBackportsShutilGetTerminalSize(PythonPackage):
homepage = "https://pypi.python.org/pypi/backports.shutil_get_terminal_size"
url = "https://pypi.io/packages/source/b/backports.shutil_get_terminal_size/backports.shutil_get_terminal_size-1.0.0.tar.gz"
py_namespace = 'backports'
version('1.0.0', '03267762480bd86b50580dc19dff3c66')
# newer setuptools version mess with "namespace" packages in an

2
var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py
View File

@@ -31,4 +31,6 @@ class PyBackportsSslMatchHostname(PythonPackage):
homepage = "https://pypi.python.org/pypi/backports.ssl_match_hostname"
url = "https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz"
py_namespace = 'backports'
version('3.5.0.1', 'c03fc5e2c7b3da46b81acf5cbacfe1e6')

50
var/spack/repos/builtin/packages/python/package.py
View File

@@ -27,10 +27,12 @@
import platform
import re
import sys
import shutil
import llnl.util.tty as tty
from llnl.util.lang import match_predicate
from llnl.util.filesystem import force_remove
from llnl.util.filesystem import (force_remove, get_filetype,
path_contains_subdirectory)
import spack.store
import spack.util.spack_json as sjson
@@ -676,33 +678,49 @@ def write_easy_install_pth(self, exts, prefix=None):
"sys.path[p:p]=new; "
"sys.__egginsert = p+len(new)\n")
def activate(self, ext_pkg, **args):
def activate(self, ext_pkg, view, **args):
ignore = self.python_ignore(ext_pkg, args)
args.update(ignore=ignore)
extensions_layout = args.get("extensions_layout",
spack.store.extensions)
super(Python, self).activate(ext_pkg, **args)
super(Python, self).activate(ext_pkg, view, **args)
extensions_layout = view.extensions_layout
exts = extensions_layout.extension_map(self.spec)
exts[ext_pkg.name] = ext_pkg.spec
self.write_easy_install_pth(
exts,
prefix=extensions_layout.extendee_target_directory(self))
self.write_easy_install_pth(exts, prefix=view.root)
def deactivate(self, ext_pkg, **args):
def deactivate(self, ext_pkg, view, **args):
args.update(ignore=self.python_ignore(ext_pkg, args))
super(Python, self).deactivate(ext_pkg, **args)
extensions_layout = args.get("extensions_layout",
spack.store.extensions)
super(Python, self).deactivate(ext_pkg, view, **args)
extensions_layout = view.extensions_layout
exts = extensions_layout.extension_map(self.spec)
# Make deactivate idempotent
if ext_pkg.name in exts:
del exts[ext_pkg.name]
self.write_easy_install_pth(
exts,
prefix=extensions_layout.extendee_target_directory(self))
self.write_easy_install_pth(exts, prefix=view.root)
def add_files_to_view(self, view, merge_map):
bin_dir = self.spec.prefix.bin
for src, dst in merge_map.items():
if not path_contains_subdirectory(src, bin_dir):
view.link(src, dst)
elif not os.path.islink(src):
shutil.copy2(src, dst)
if 'script' in get_filetype(src):
filter_file(
self.spec.prefix, os.path.abspath(view.root), dst)
else:
orig_link_target = os.path.realpath(src)
new_link_target = os.path.abspath(merge_map[orig_link_target])
view.link(new_link_target, dst)
def remove_files_from_view(self, view, merge_map):
bin_dir = self.spec.prefix.bin
for src, dst in merge_map.items():
if not path_contains_subdirectory(src, bin_dir):
view.remove_file(src, dst)
else:
os.remove(dst)