zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/LLNL/spack into refactoring/stage

Browse Source 
Conflicts:
	lib/spack/spack/package.py
This commit is contained in:
alalazo
2016-03-03 10:19:45 +01:00
parent 5ef3621556 6701977f1a
commit 7f4a69319a
24 changed files with 238 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/docs/basic_usage.rst
View File

@@ -896,7 +896,7 @@ Or, similarly with modules, you could type:
$ spack load mpich %gcc@4.4.7
These commands will add appropriate directories to your ``PATH``,
``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use
``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use
a package, you can type unload or unuse similarly:
.. code-block:: sh

2
lib/spack/docs/getting_started.rst
View File

@@ -22,7 +22,7 @@ go:
$ spack install libelf
For a richer experience, use Spack's `shell support
<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_:
<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_:
.. code-block:: sh

2
lib/spack/env/cc vendored
View File

@@ -130,7 +130,7 @@ if [ -z "$mode" ]; then
done
fi
# Dump the version and exist if we're in testing mode.
# Dump the version and exit if we're in testing mode.
if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then
echo "$mode"
exit

11
lib/spack/llnl/util/filesystem.py
View File

@@ -152,15 +152,20 @@ def set_install_permissions(path):
def copy_mode(src, dest):
src_mode = os.stat(src).st_mode
dest_mode = os.stat(dest).st_mode
if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
os.chmod(dest, dest_mode)
def install(src, dest):
"""Manually install a file to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
# Expand dsst to its eventual full path if it is a directory.
if os.path.isdir(dest):
dest = join_path(dest, os.path.basename(src))
shutil.copy(src, dest)
set_install_permissions(dest)
copy_mode(src, dest)

13
lib/spack/spack/cmd/checksum.py
View File

@@ -58,24 +58,29 @@ def get_checksums(versions, urls, **kwargs):
tty.msg("Downloading...")
hashes = []
for i, (url, version) in enumerate(zip(urls, versions)):
i = 0
for url, version in zip(urls, versions):
stage = Stage(url)
try:
stage.fetch()
if i == 0 and first_stage_function:
first_stage_function(stage)
hashes.append(
spack.util.crypto.checksum(hashlib.md5, stage.archive_file))
hashes.append((version,
spack.util.crypto.checksum(hashlib.md5, stage.archive_file)))
except FailedDownloadError, e:
tty.msg("Failed to fetch %s" % url)
continue
except Exception, e:
tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e))
continue
finally:
if not keep_stage:
stage.destroy()
i += 1
return zip(versions, hashes)
return hashes

48
lib/spack/spack/cmd/repo.py
View File

@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://llnl.github.io/spack
# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -74,51 +74,7 @@ def setup_parser(subparser):
def repo_create(args):
"""Create a new package repository."""
root = canonicalize_path(args.directory)
namespace = args.namespace
if not args.namespace:
namespace = os.path.basename(root)
if not re.match(r'\w[\.\w-]*', namespace):
tty.die("'%s' is not a valid namespace." % namespace)
existed = False
if os.path.exists(root):
if os.path.isfile(root):
tty.die('File %s already exists and is not a directory' % root)
elif os.path.isdir(root):
if not os.access(root, os.R_OK | os.W_OK):
tty.die('Cannot create new repo in %s: cannot access directory.' % root)
if os.listdir(root):
tty.die('Cannot create new repo in %s: directory is not empty.' % root)
existed = True
full_path = os.path.realpath(root)
parent = os.path.dirname(full_path)
if not os.access(parent, os.R_OK | os.W_OK):
tty.die("Cannot create repository in %s: can't access parent!" % root)
try:
config_path = os.path.join(root, repo_config_name)
packages_path = os.path.join(root, packages_dir_name)
mkdirp(packages_path)
with open(config_path, 'w') as config:
config.write("repo:\n")
config.write(" namespace: '%s'\n" % namespace)
except (IOError, OSError) as e:
tty.die('Failed to create new repository in %s.' % root,
"Caused by %s: %s" % (type(e), e))
# try to clean up.
if existed:
shutil.rmtree(config_path, ignore_errors=True)
shutil.rmtree(packages_path, ignore_errors=True)
else:
shutil.rmtree(root, ignore_errors=True)
full_path, namespace = create_repo(args.directory, args.namespace)
tty.msg("Created repo with namespace '%s'." % namespace)
tty.msg("To register it with spack, run this command:",
'spack repo add %s' % full_path)

6
lib/spack/spack/directives.py
View File

@@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs):
def _depends_on(pkg, spec, when=None):
if when is None:
# If when is False do nothing
if when is False:
return
# If when is None or True make sure the condition is always satisfied
if when is None or when is True:
when = pkg.name
when_spec = parse_anonymous_spec(when, pkg.name)

14
lib/spack/spack/directory_layout.py
View File

@@ -173,7 +173,9 @@ def __init__(self, root, **kwargs):
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
self.build_log_name = 'build.out' # TODO: use config file.
self.build_log_name = 'build.out' # build log.
self.build_env_name = 'build.env' # build environment
self.packages_dir = 'repos' # archive of package.py files
# Cache of already written/read extension maps.
self._extension_maps = {}
@@ -231,6 +233,16 @@ def build_log_path(self, spec):
self.build_log_name)
def build_env_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir,
self.build_env_name)
def build_packages_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir,
self.packages_dir)
def create_install_directory(self, spec):
_check_concrete(spec)

2
lib/spack/spack/modules.py
View File

@@ -33,6 +33,7 @@
* /bin directories to be appended to PATH
* /lib* directories for LD_LIBRARY_PATH
* /include directories for CPATH
* /man* and /share/man* directories for MANPATH
* the package prefix for CMAKE_PREFIX_PATH
@@ -121,6 +122,7 @@ def add_path(path_name, directory):
('LIBRARY_PATH', self.spec.prefix.lib64),
('LD_LIBRARY_PATH', self.spec.prefix.lib),
('LD_LIBRARY_PATH', self.spec.prefix.lib64),
('CPATH', self.spec.prefix.include),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')),
('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]:

9
lib/spack/spack/multimethod.py
View File

@@ -193,10 +193,11 @@ def install(self, prefix):
platform-specific versions. There's not much we can do to get
around this because of the way decorators work.
"""
class when(object):
def __init__(self, spec):
pkg = get_calling_module_name()
self.spec = parse_anonymous_spec(spec, pkg)
if spec is True:
spec = pkg
self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None
def __call__(self, method):
# Get the first definition of the method in the calling scope
@@ -207,7 +208,9 @@ def __call__(self, method):
if not type(original_method) == SpecMultiMethod:
original_method = SpecMultiMethod(original_method)
original_method.register(self.spec, method)
if self.spec is not None:
original_method.register(self.spec, method)
return original_method

57
lib/spack/spack/package.py
View File

@@ -58,6 +58,7 @@
import spack.mirror
import spack.hooks
import spack.directives
import spack.repository
import spack.build_environment
import spack.url
import spack.util.web
@@ -66,6 +67,7 @@
from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
from spack.util.environment import dump_environment
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -884,10 +886,14 @@ def real_work():
# Do the real install in the source directory.
self.stage.chdir_to_source()
# Save the build environment in a file before building.
env_path = join_path(os.getcwd(), 'spack-build.env')
# This redirects I/O to a build log (and optionally to the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w')
with log_output(log_file, verbose, sys.stdout.isatty(), True):
dump_environment(env_path)
self.install(self.spec, self.prefix)
# Ensure that something was actually installed.
@@ -896,7 +902,12 @@ def real_work():
# Move build log into install directory on success
if not fake:
log_install_path = spack.install_layout.build_log_path(self.spec)
env_install_path = spack.install_layout.build_env_path(self.spec)
install(log_path, log_install_path)
install(env_path, env_install_path)
packages_dir = spack.install_layout.build_packages_path(self.spec)
dump_packages(self.spec, packages_dir)
# On successful install, remove the stage.
if not keep_stage:
@@ -1205,6 +1216,52 @@ def validate_package_url(url_string):
tty.die("Invalid file type in URL: '%s'" % url_string)
def dump_packages(spec, path):
"""Dump all package information for a spec and its dependencies.
This creates a package repository within path for every
namespace in the spec DAG, and fills the repos wtih package
files and patch files for every node in the DAG.
"""
mkdirp(path)
# Copy in package.py files from any dependencies.
# Note that we copy them in as they are in the *install* directory
# NOT as they are in the repository, because we want a snapshot of
# how *this* particular build was done.
for node in spec.traverse():
if node is not spec:
# Locate the dependency package in the install tree and find
# its provenance information.
source = spack.install_layout.build_packages_path(node)
source_repo_root = join_path(source, node.namespace)
# There's no provenance installed for the source package. Skip it.
# User can always get something current from the builtin repo.
if not os.path.isdir(source_repo_root):
continue
# Create a source repo and get the pkg directory out of it.
try:
source_repo = spack.repository.Repo(source_repo_root)
source_pkg_dir = source_repo.dirname_for_package_name(node.name)
except RepoError as e:
tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
# Create a destination repository
dest_repo_root = join_path(path, node.namespace)
if not os.path.exists(dest_repo_root):
spack.repository.create_repo(dest_repo_root)
repo = spack.repository.Repo(dest_repo_root)
# Get the location of the package in the dest repo.
dest_pkg_dir = repo.dirname_for_package_name(node.name)
if node is not spec:
install_tree(source_pkg_dir, dest_pkg_dir)
else:
spack.repo.dump_provenance(node, dest_pkg_dir)
def print_pkg(message):
"""Outputs a message with a package icon."""
from llnl.util.tty.color import cwrite

99
lib/spack/spack/repository.py
View File

@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://llnl.github.io/spack
# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -33,7 +33,7 @@
from external import yaml
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
from llnl.util.filesystem import *
import spack.error
import spack.config
@@ -316,6 +316,16 @@ def get(self, spec, new=False):
return self.repo_for_pkg(spec).get(spec)
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).dump_provenance(spec, path)
def dirname_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
@@ -552,6 +562,35 @@ def get(self, spec, new=False):
return self._instances[key]
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
# Some preliminary checks.
if spec.virtual:
raise UnknownPackageError(spec.name)
if spec.namespace and spec.namespace != self.namespace:
raise UnknownPackageError("Repository %s does not contain package %s."
% (self.namespace, spec.fullname))
# Install any patch files needed by packages.
mkdirp(path)
for spec, patches in spec.package.patches.items():
for patch in patches:
if patch.path:
if os.path.exists(patch.path):
install(patch.path, path)
else:
tty.warn("Patch file did not exist: %s" % patch.path)
# Install the package.py file itself.
install(self.filename_for_package_name(spec), path)
def purge(self):
"""Clear entire package instance cache."""
self._instances.clear()
@@ -705,6 +744,58 @@ def __contains__(self, pkg_name):
return self.exists(pkg_name)
def create_repo(root, namespace=None):
"""Create a new repository in root with the specified namespace.
If the namespace is not provided, use basename of root.
Return the canonicalized path and the namespace of the created repository.
"""
root = canonicalize_path(root)
if not namespace:
namespace = os.path.basename(root)
if not re.match(r'\w[\.\w-]*', namespace):
raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
existed = False
if os.path.exists(root):
if os.path.isfile(root):
raise BadRepoError('File %s already exists and is not a directory' % root)
elif os.path.isdir(root):
if not os.access(root, os.R_OK | os.W_OK):
raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
if os.listdir(root):
raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
existed = True
full_path = os.path.realpath(root)
parent = os.path.dirname(full_path)
if not os.access(parent, os.R_OK | os.W_OK):
raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
try:
config_path = os.path.join(root, repo_config_name)
packages_path = os.path.join(root, packages_dir_name)
mkdirp(packages_path)
with open(config_path, 'w') as config:
config.write("repo:\n")
config.write(" namespace: '%s'\n" % namespace)
except (IOError, OSError) as e:
raise BadRepoError('Failed to create new repository in %s.' % root,
"Caused by %s: %s" % (type(e), e))
# try to clean up.
if existed:
shutil.rmtree(config_path, ignore_errors=True)
shutil.rmtree(packages_path, ignore_errors=True)
else:
shutil.rmtree(root, ignore_errors=True)
return full_path, namespace
class RepoError(spack.error.SpackError):
"""Superclass for repository-related errors."""
@@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError):
"""Raised when there are no repositories configured."""
class InvalidNamespaceError(RepoError):
"""Raised when an invalid namespace is encountered."""
class BadRepoError(RepoError):
"""Raised when repo layout is invalid."""

2
lib/spack/spack/resource.py
View File

@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://llnl.github.io/spack
# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify

2
lib/spack/spack/test/namespace_trie.py
View File

@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://llnl.github.io/spack
# For details, see https://software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify

8
lib/spack/spack/test/tally_plugin.py
View File

@@ -6,7 +6,7 @@
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# For details, see https://scalability-software.llnl.gov/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
@@ -34,7 +34,7 @@ def __init__(self):
self.successCount = 0
self.failCount = 0
self.errorCount = 0
@property
def numberOfTestsRun(self):
"""Excludes skipped tests"""
@@ -48,10 +48,10 @@ def configure(self, options, conf):
def addSuccess(self, test):
self.successCount += 1
def addError(self, test, err):
self.errorCount += 1
def addFailure(self, test, err):
self.failCount += 1

7
lib/spack/spack/util/environment.py
View File

@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
dictionary.pop(key)
def dump_environment(path):
"""Dump the current environment out to a file."""
with open(path, 'w') as env_file:
for key,val in sorted(os.environ.items()):
env_file.write("%s=%s\n" % (key, val))

6
lib/spack/spack/util/web.py
View File

@@ -86,12 +86,12 @@ def _spider(args):
if not "Content-type" in resp.headers:
tty.debug("ignoring page " + url)
return pages
return pages, links
if not resp.headers["Content-type"].startswith('text/html'):
tty.debug("ignoring page " + url + " with content type " +
resp.headers["Content-type"])
return pages
return pages, links
# Do the real GET request when we know it's just HTML.
req.get_method = lambda: "GET"
@@ -173,7 +173,7 @@ def spider(root_url, **kwargs):
performance over a sequential fetch.
"""
max_depth = kwargs.setdefault('depth', 1)
pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
return pages, links