zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into env-draco

Browse Source
This commit is contained in:
Kelly (KT) Thompson
2016-04-04 13:57:03 -06:00
parent 9bb99de0a7 220fb068aa
commit aef7065e85
26 changed files with 730 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
lib/spack/docs/basic_usage.rst
View File

@@ -149,26 +149,46 @@ customize an installation in :ref:`sec-specs`.
``spack uninstall``
~~~~~~~~~~~~~~~~~~~~~
To uninstall a package, type ``spack uninstall <package>``. This will
completely remove the directory in which the package was installed.
To uninstall a package, type ``spack uninstall <package>``. This will ask the user for
confirmation, and in case will completely remove the directory in which the package was installed.
.. code-block:: sh
spack uninstall mpich
If there are still installed packages that depend on the package to be
uninstalled, spack will refuse to uninstall it. You can override this
behavior with ``spack uninstall -f <package>``, but you risk breaking
other installed packages. In general, it is safer to remove dependent
packages *before* removing their dependencies.
uninstalled, spack will refuse to uninstall it.
A line like ``spack uninstall mpich`` may be ambiguous, if multiple
``mpich`` configurations are installed. For example, if both
To uninstall a package and every package that depends on it, you may give the
`--dependents` option.
.. code-block:: sh
spack uninstall --dependents mpich
will display a list of all the packages that depends on `mpich` and, upon confirmation,
will uninstall them in the right order.
A line like
.. code-block:: sh
spack uninstall mpich
may be ambiguous, if multiple ``mpich`` configurations are installed. For example, if both
``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer
to either one. Because it cannot determine which one to uninstall,
Spack will ask you to provide a version number to remove the
ambiguity. As an example, ``spack uninstall mpich@3.1`` is
unambiguous in this scenario.
Spack will ask you either to provide a version number to remove the
ambiguity or use the ``--all`` option to uninstall all of the matching packages.
You may force uninstall a package with the `--force` option
.. code-block:: sh
spack uninstall --force mpich
but you risk breaking other installed packages. In general, it is safer to remove dependent
packages *before* removing their dependencies or use the `--dependents` option.
Seeing installed packages
@@ -774,6 +794,34 @@ Environment modules
Spack provides some limited integration with environment module
systems to make it easier to use the packages it provides.
Installing Environment Modules
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In order to use Spack's generated environment modules, you must have
installed the *Environment Modules* package. On many Linux
distributions, this can be installed from the vendor's repository.
For example: ```yum install environment-modules``
(Fedora/RHEL/CentOS). If your Linux distribution does not have
Environment Modules, you can get it with Spack:
1. Install with::
spack install environment-modules
2. Activate with::
MODULES_HOME=`spack location -i environment-modules`
MODULES_VERSION=`ls -1 $MODULES_HOME/Modules | head -1`
${MODULES_HOME}/Modules/${MODULES_VERSION}/bin/add.modules
This adds to your ``.bashrc`` (or similar) files, enabling Environment
Modules when you log in. It will ask your permission before changing
any files.
Spack and Environment Modules
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can enable shell support by sourcing some files in the
``/share/spack`` directory.

41
lib/spack/spack/build_environment.py
View File

@@ -213,7 +213,7 @@ def set_module_variables_for_package(pkg, module):
# TODO: of build dependencies, as opposed to link dependencies.
# TODO: Currently, everything is a link dependency, but tools like
# TODO: this shouldn't be.
m.cmake = which("cmake")
m.cmake = Executable('cmake')
# standard CMake arguments
m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix,
@@ -278,21 +278,6 @@ def parent_class_modules(cls):
return result
def setup_module_variables_for_dag(pkg):
"""Set module-scope variables for all packages in the DAG."""
for spec in pkg.spec.traverse(order='post'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = spec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
def setup_package(pkg):
"""Execute all environment setup routines."""
spack_env = EnvironmentModifications()
@@ -316,20 +301,26 @@ def setup_package(pkg):
set_compiler_environment_variables(pkg, spack_env)
set_build_environment_variables(pkg, spack_env)
setup_module_variables_for_dag(pkg)
# Allow dependencies to modify the module
# traverse in postorder so package can use vars from its dependencies
spec = pkg.spec
for dependency_spec in spec.traverse(root=False):
dpkg = dependency_spec.package
dpkg.setup_dependent_package(pkg.module, spec)
for dspec in pkg.spec.traverse(order='post'):
# If a user makes their own package repo, e.g.
# spack.repos.mystuff.libelf.Libelf, and they inherit from
# an existing class like spack.repos.original.libelf.Libelf,
# then set the module variables for both classes so the
# parent class can still use them if it gets called.
spkg = dspec.package
modules = parent_class_modules(spkg.__class__)
for mod in modules:
set_module_variables_for_package(spkg, mod)
set_module_variables_for_package(spkg, spkg.module)
# Allow dependencies to set up environment as well
for dependency_spec in spec.traverse(root=False):
dpkg = dependency_spec.package
# Allow dependencies to modify the module
dpkg = dspec.package
dpkg.setup_dependent_package(pkg.module, spec)
dpkg.setup_dependent_environment(spack_env, run_env, spec)
# Allow the package to apply some settings.
pkg.setup_environment(spack_env, run_env)
# Make sure nothing's strange about the Spack environment.

191
lib/spack/spack/cmd/uninstall.py
View File

@@ -23,19 +23,33 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from __future__ import print_function
import sys
import argparse
import llnl.util.tty as tty
from llnl.util.tty.colify import colify
import spack
import spack.cmd
import spack.repository
from spack.cmd.find import display_specs
from spack.package import PackageStillNeededError
description="Remove an installed package"
description = "Remove an installed package"
error_message = """You can either:
a) Use a more specific spec, or
b) use spack uninstall -a to uninstall ALL matching specs.
"""
def ask_for_confirmation(message):
while True:
tty.msg(message + '[y/n]')
choice = raw_input().lower()
if choice == 'y':
break
elif choice == 'n':
raise SystemExit('Operation aborted')
tty.warn('Please reply either "y" or "n"')
def setup_parser(subparser):
subparser.add_argument(
@@ -44,10 +58,101 @@ def setup_parser(subparser):
subparser.add_argument(
'-a', '--all', action='store_true', dest='all',
help="USE CAREFULLY. Remove ALL installed packages that match each " +
"supplied spec. i.e., if you say uninstall libelf, ALL versions of " +
"libelf are uninstalled. This is both useful and dangerous, like rm -r.")
"supplied spec. i.e., if you say uninstall libelf, ALL versions of " +
"libelf are uninstalled. This is both useful and dangerous, like rm -r.")
subparser.add_argument(
'packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall")
'-d', '--dependents', action='store_true', dest='dependents',
help='Also uninstall any packages that depend on the ones given via command line.'
)
subparser.add_argument(
'-y', '--yes-to-all', action='store_true', dest='yes_to_all',
help='Assume "yes" is the answer to every confirmation asked to the user.'
)
subparser.add_argument('packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall")
def concretize_specs(specs, allow_multiple_matches=False, force=False):
"""
Returns a list of specs matching the non necessarily concretized specs given from cli
Args:
specs: list of specs to be matched against installed packages
allow_multiple_matches : boolean (if True multiple matches for each item in specs are admitted)
Return:
list of specs
"""
specs_from_cli = [] # List of specs that match expressions given via command line
has_errors = False
for spec in specs:
matching = spack.installed_db.query(spec)
# For each spec provided, make sure it refers to only one package.
# Fail and ask user to be unambiguous if it doesn't
if not allow_multiple_matches and len(matching) > 1:
tty.error("%s matches multiple packages:" % spec)
print()
display_specs(matching, long=True)
print()
has_errors = True
# No installed package matches the query
if len(matching) == 0 and not force:
tty.error("%s does not match any installed packages." % spec)
has_errors = True
specs_from_cli.extend(matching)
if has_errors:
tty.die(error_message)
return specs_from_cli
def installed_dependents(specs):
"""
Returns a dictionary that maps a spec with a list of its installed dependents
Args:
specs: list of specs to be checked for dependents
Returns:
dictionary of installed dependents
"""
dependents = {}
for item in specs:
lst = [x for x in item.package.installed_dependents if x not in specs]
if lst:
lst = list(set(lst))
dependents[item] = lst
return dependents
def do_uninstall(specs, force):
"""
Uninstalls all the specs in a list.
Args:
specs: list of specs to be uninstalled
force: force uninstallation (boolean)
"""
packages = []
for item in specs:
try:
# should work if package is known to spack
packages.append(item.package)
except spack.repository.UnknownPackageError as e:
# The package.py file has gone away -- but still
# want to uninstall.
spack.Package(item).do_uninstall(force=True)
# Sort packages to be uninstalled by the number of installed dependents
# This ensures we do things in the right order
def num_installed_deps(pkg):
return len(pkg.installed_dependents)
packages.sort(key=num_installed_deps)
for item in packages:
item.do_uninstall(force=force)
def uninstall(parser, args):
@@ -56,50 +161,34 @@ def uninstall(parser, args):
with spack.installed_db.write_transaction():
specs = spack.cmd.parse_specs(args.packages)
# Gets the list of installed specs that match the ones give via cli
uninstall_list = concretize_specs(specs, args.all, args.force) # takes care of '-a' is given in the cli
dependent_list = installed_dependents(uninstall_list) # takes care of '-d'
# For each spec provided, make sure it refers to only one package.
# Fail and ask user to be unambiguous if it doesn't
pkgs = []
for spec in specs:
matching_specs = spack.installed_db.query(spec)
if not args.all and len(matching_specs) > 1:
tty.error("%s matches multiple packages:" % spec)
print()
display_specs(matching_specs, long=True)
print()
print("You can either:")
print(" a) Use a more specific spec, or")
print(" b) use spack uninstall -a to uninstall ALL matching specs.")
sys.exit(1)
if len(matching_specs) == 0:
if args.force: continue
tty.die("%s does not match any installed packages." % spec)
for s in matching_specs:
try:
# should work if package is known to spack
pkgs.append(s.package)
except spack.repository.UnknownPackageError as e:
# The package.py file has gone away -- but still
# want to uninstall.
spack.Package(s).do_uninstall(force=True)
# Sort packages to be uninstalled by the number of installed dependents
# This ensures we do things in the right order
def num_installed_deps(pkg):
return len(pkg.installed_dependents)
pkgs.sort(key=num_installed_deps)
# Uninstall packages in order now.
for pkg in pkgs:
try:
pkg.do_uninstall(force=args.force)
except PackageStillNeededError as e:
tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True))
# Process dependent_list and update uninstall_list
has_error = False
if dependent_list and not args.dependents and not args.force:
for spec, lst in dependent_list.items():
tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True))
print('')
print("The following packages depend on it:")
display_specs(e.dependents, long=True)
display_specs(lst, long=True)
print('')
print("You can use spack uninstall -f to force this action.")
sys.exit(1)
has_error = True
elif args.dependents:
for key, lst in dependent_list.items():
uninstall_list.extend(lst)
uninstall_list = list(set(uninstall_list))
if has_error:
tty.die('You can use spack uninstall --dependents to uninstall these dependencies as well')
if not args.yes_to_all:
tty.msg("The following packages will be uninstalled : ")
print('')
display_specs(uninstall_list, long=True)
print('')
ask_for_confirmation('Do you want to proceed ? ')
# Uninstall everything on the list
do_uninstall(uninstall_list, args.force)

11
lib/spack/spack/modules.py
View File

@@ -163,9 +163,14 @@ def write(self):
# package-specific modifications
spack_env = EnvironmentModifications()
for item in self.pkg.extendees:
package = self.spec[item].package
package.setup_dependent_package(self.pkg.module, self.spec)
package.setup_dependent_environment(spack_env, env, self.spec)
try:
package = self.spec[item].package
package.setup_dependent_package(self.pkg.module, self.spec)
package.setup_dependent_environment(spack_env, env, self.spec)
except:
# The extends was conditional, so it doesn't count here
# eg: extends('python', when='+python')
pass
# Package-specific environment modifications
self.spec.package.setup_environment(spack_env, env)

3
lib/spack/spack/test/__init__.py
View File

@@ -67,7 +67,8 @@
'namespace_trie',
'yaml',
'sbang',
'environment']
'environment',
'cmd.uninstall']
def list_tests():

0
lib/spack/spack/test/cmd/__init__.py Normal file
View File

37
lib/spack/spack/test/cmd/uninstall.py Normal file
View File

@@ -0,0 +1,37 @@
import spack.test.mock_database
from spack.cmd.uninstall import uninstall
class MockArgs(object):
def __init__(self, packages, all=False, force=False, dependents=False):
self.packages = packages
self.all = all
self.force = force
self.dependents = dependents
self.yes_to_all = True
class TestUninstall(spack.test.mock_database.MockDatabase):
def test_uninstall(self):
parser = None
# Multiple matches
args = MockArgs(['mpileaks'])
self.assertRaises(SystemExit, uninstall, parser, args)
# Installed dependents
args = MockArgs(['libelf'])
self.assertRaises(SystemExit, uninstall, parser, args)
# Recursive uninstall
args = MockArgs(['callpath'], all=True, dependents=True)
uninstall(parser, args)
all_specs = spack.install_layout.all_specs()
self.assertEqual(len(all_specs), 7)
# query specs with multiple configurations
mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
callpath_specs = [s for s in all_specs if s.satisfies('callpath')]
mpi_specs = [s for s in all_specs if s.satisfies('mpi')]
self.assertEqual(len(mpileaks_specs), 0)
self.assertEqual(len(callpath_specs), 0)
self.assertEqual(len(mpi_specs), 3)

82
lib/spack/spack/test/database.py
View File

@@ -28,16 +28,12 @@
"""
import os.path
import multiprocessing
import shutil
import tempfile
import spack
from llnl.util.filesystem import join_path
from llnl.util.lock import *
from llnl.util.tty.colify import colify
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
from spack.test.mock_packages_test import *
from spack.test.mock_database import MockDatabase
def _print_ref_counts():
@@ -75,80 +71,7 @@ def add_rec(spec):
colify(recs, cols=3)
class DatabaseTest(MockPackagesTest):
def _mock_install(self, spec):
s = Spec(spec)
s.concretize()
pkg = spack.repo.get(s)
pkg.do_install(fake=True)
def _mock_remove(self, spec):
specs = spack.installed_db.query(spec)
assert(len(specs) == 1)
spec = specs[0]
spec.package.do_uninstall(spec)
def setUp(self):
super(DatabaseTest, self).setUp()
#
# TODO: make the mockup below easier.
#
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
# across each install.
#
# Here is what the mock DB looks like:
#
# o mpileaks o mpileaks' o mpileaks''
# |\ |\ |\
# | o callpath | o callpath' | o callpath''
# |/| |/| |/|
# o | mpich o | mpich2 o | zmpi
# | | o | fake
# | | |
# | |______________/
# | .____________/
# |/
# o dyninst
# |\
# | o libdwarf
# |/
# o libelf
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def tearDown(self):
super(DatabaseTest, self).tearDown()
shutil.rmtree(self.install_path)
spack.install_path = self.spack_install_path
spack.install_layout = self.spack_install_layout
spack.installed_db = self.spack_installed_db
class DatabaseTest(MockDatabase):
def test_005_db_exists(self):
"""Make sure db cache file exists after creating."""
index_file = join_path(self.install_path, '.spack-db', 'index.yaml')
@@ -157,7 +80,6 @@ def test_005_db_exists(self):
self.assertTrue(os.path.exists(index_file))
self.assertTrue(os.path.exists(lock_file))
def test_010_all_install_sanity(self):
"""Ensure that the install layout reflects what we think it does."""
all_specs = spack.install_layout.all_specs()

29
lib/spack/spack/test/install.py
View File

@@ -64,7 +64,14 @@ def tearDown(self):
shutil.rmtree(self.tmpdir, ignore_errors=True)
def test_install_and_uninstall(self):
def fake_fetchify(self, pkg):
"""Fake the URL for a package so it downloads from a file."""
fetcher = FetchStrategyComposite()
fetcher.append(URLFetchStrategy(self.repo.url))
pkg.fetcher = fetcher
def ztest_install_and_uninstall(self):
# Get a basic concrete spec for the trivial install package.
spec = Spec('trivial_install_test_package')
spec.concretize()
@@ -73,11 +80,7 @@ def test_install_and_uninstall(self):
# Get the package
pkg = spack.repo.get(spec)
# Fake the URL for the package so it downloads from a file.
fetcher = FetchStrategyComposite()
fetcher.append(URLFetchStrategy(self.repo.url))
pkg.fetcher = fetcher
self.fake_fetchify(pkg)
try:
pkg.do_install()
@@ -85,3 +88,17 @@ def test_install_and_uninstall(self):
except Exception, e:
pkg.remove_prefix()
raise
def test_install_environment(self):
spec = Spec('cmake-client').concretized()
for s in spec.traverse():
self.fake_fetchify(s.package)
pkg = spec.package
try:
pkg.do_install()
except Exception, e:
pkg.remove_prefix()
raise

78
lib/spack/spack/test/mock_database.py Normal file
View File

@@ -0,0 +1,78 @@
import shutil
import tempfile
import spack
from spack.spec import Spec
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
from spack.test.mock_packages_test import MockPackagesTest
class MockDatabase(MockPackagesTest):
def _mock_install(self, spec):
s = Spec(spec)
s.concretize()
pkg = spack.repo.get(s)
pkg.do_install(fake=True)
def _mock_remove(self, spec):
specs = spack.installed_db.query(spec)
assert(len(specs) == 1)
spec = specs[0]
spec.package.do_uninstall(spec)
def setUp(self):
super(MockDatabase, self).setUp()
#
# TODO: make the mockup below easier.
#
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
# across each install.
#
# Here is what the mock DB looks like:
#
# o mpileaks o mpileaks' o mpileaks''
# |\ |\ |\
# | o callpath | o callpath' | o callpath''
# |/| |/| |/|
# o | mpich o | mpich2 o | zmpi
# | | o | fake
# | | |
# | |______________/
# | .____________/
# |/
# o dyninst
# |\
# | o libdwarf
# |/
# o libelf
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def tearDown(self):
super(MockDatabase, self).tearDown()
shutil.rmtree(self.install_path)
spack.install_path = self.spack_install_path
spack.install_layout = self.spack_install_layout
spack.installed_db = self.spack_installed_db