zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #475 from LLNL/features/env-and-package-provenance

Browse Source 
Features/env and package provenance
This commit is contained in:
Todd Gamblin
2016-03-03 00:42:40 -08:00
parent 51f60f3a1e 52081c46d6
commit 6701977f1a
6 changed files with 183 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/spack/llnl/util/filesystem.py
View File

@@ -152,15 +152,20 @@ def set_install_permissions(path):
def copy_mode(src, dest):
src_mode = os.stat(src).st_mode
dest_mode = os.stat(dest).st_mode
if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
os.chmod(dest, dest_mode)
def install(src, dest):
"""Manually install a file to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
# Expand dsst to its eventual full path if it is a directory.
if os.path.isdir(dest):
dest = join_path(dest, os.path.basename(src))
shutil.copy(src, dest)
set_install_permissions(dest)
copy_mode(src, dest)

46
lib/spack/spack/cmd/repo.py
View File

@@ -74,51 +74,7 @@ def setup_parser(subparser):
def repo_create(args):
"""Create a new package repository."""
root = canonicalize_path(args.directory)
namespace = args.namespace
if not args.namespace:
namespace = os.path.basename(root)
if not re.match(r'\w[\.\w-]*', namespace):
tty.die("'%s' is not a valid namespace." % namespace)
existed = False
if os.path.exists(root):
if os.path.isfile(root):
tty.die('File %s already exists and is not a directory' % root)
elif os.path.isdir(root):
if not os.access(root, os.R_OK | os.W_OK):
tty.die('Cannot create new repo in %s: cannot access directory.' % root)
if os.listdir(root):
tty.die('Cannot create new repo in %s: directory is not empty.' % root)
existed = True
full_path = os.path.realpath(root)
parent = os.path.dirname(full_path)
if not os.access(parent, os.R_OK | os.W_OK):
tty.die("Cannot create repository in %s: can't access parent!" % root)
try:
config_path = os.path.join(root, repo_config_name)
packages_path = os.path.join(root, packages_dir_name)
mkdirp(packages_path)
with open(config_path, 'w') as config:
config.write("repo:\n")
config.write(" namespace: '%s'\n" % namespace)
except (IOError, OSError) as e:
tty.die('Failed to create new repository in %s.' % root,
"Caused by %s: %s" % (type(e), e))
# try to clean up.
if existed:
shutil.rmtree(config_path, ignore_errors=True)
shutil.rmtree(packages_path, ignore_errors=True)
else:
shutil.rmtree(root, ignore_errors=True)
full_path, namespace = create_repo(args.directory, args.namespace)
tty.msg("Created repo with namespace '%s'." % namespace)
tty.msg("To register it with spack, run this command:",
'spack repo add %s' % full_path)

14
lib/spack/spack/directory_layout.py
View File

@@ -173,7 +173,9 @@ def __init__(self, root, **kwargs):
self.spec_file_name = 'spec.yaml'
self.extension_file_name = 'extensions.yaml'
self.build_log_name = 'build.out' # TODO: use config file.
self.build_log_name = 'build.out' # build log.
self.build_env_name = 'build.env' # build environment
self.packages_dir = 'repos' # archive of package.py files
# Cache of already written/read extension maps.
self._extension_maps = {}
@@ -231,6 +233,16 @@ def build_log_path(self, spec):
self.build_log_name)
def build_env_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir,
self.build_env_name)
def build_packages_path(self, spec):
return join_path(self.path_for_spec(spec), self.metadata_dir,
self.packages_dir)
def create_install_directory(self, spec):
_check_concrete(spec)

58
lib/spack/spack/package.py
View File

@@ -58,6 +58,7 @@
import spack.mirror
import spack.hooks
import spack.directives
import spack.repository
import spack.build_environment
import spack.url
import spack.util.web
@@ -66,6 +67,7 @@
from spack.stage import Stage, ResourceStage, StageComposite
from spack.util.compression import allowed_archive, extension
from spack.util.executable import ProcessError
from spack.util.environment import dump_environment
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -501,6 +503,7 @@ def fetcher(self):
self._fetcher = self._make_fetcher()
return self._fetcher
@fetcher.setter
def fetcher(self, f):
self._fetcher = f
@@ -884,10 +887,14 @@ def real_work():
# Do the real install in the source directory.
self.stage.chdir_to_source()
# Save the build environment in a file before building.
env_path = join_path(os.getcwd(), 'spack-build.env')
# This redirects I/O to a build log (and optionally to the terminal)
log_path = join_path(os.getcwd(), 'spack-build.out')
log_file = open(log_path, 'w')
with log_output(log_file, verbose, sys.stdout.isatty(), True):
dump_environment(env_path)
self.install(self.spec, self.prefix)
# Ensure that something was actually installed.
@@ -896,7 +903,12 @@ def real_work():
# Move build log into install directory on success
if not fake:
log_install_path = spack.install_layout.build_log_path(self.spec)
env_install_path = spack.install_layout.build_env_path(self.spec)
install(log_path, log_install_path)
install(env_path, env_install_path)
packages_dir = spack.install_layout.build_packages_path(self.spec)
dump_packages(self.spec, packages_dir)
# On successful install, remove the stage.
if not keep_stage:
@@ -1212,6 +1224,52 @@ def validate_package_url(url_string):
tty.die("Invalid file type in URL: '%s'" % url_string)
def dump_packages(spec, path):
"""Dump all package information for a spec and its dependencies.
This creates a package repository within path for every
namespace in the spec DAG, and fills the repos wtih package
files and patch files for every node in the DAG.
"""
mkdirp(path)
# Copy in package.py files from any dependencies.
# Note that we copy them in as they are in the *install* directory
# NOT as they are in the repository, because we want a snapshot of
# how *this* particular build was done.
for node in spec.traverse():
if node is not spec:
# Locate the dependency package in the install tree and find
# its provenance information.
source = spack.install_layout.build_packages_path(node)
source_repo_root = join_path(source, node.namespace)
# There's no provenance installed for the source package. Skip it.
# User can always get something current from the builtin repo.
if not os.path.isdir(source_repo_root):
continue
# Create a source repo and get the pkg directory out of it.
try:
source_repo = spack.repository.Repo(source_repo_root)
source_pkg_dir = source_repo.dirname_for_package_name(node.name)
except RepoError as e:
tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
# Create a destination repository
dest_repo_root = join_path(path, node.namespace)
if not os.path.exists(dest_repo_root):
spack.repository.create_repo(dest_repo_root)
repo = spack.repository.Repo(dest_repo_root)
# Get the location of the package in the dest repo.
dest_pkg_dir = repo.dirname_for_package_name(node.name)
if node is not spec:
install_tree(source_pkg_dir, dest_pkg_dir)
else:
spack.repo.dump_provenance(node, dest_pkg_dir)
def print_pkg(message):
"""Outputs a message with a package icon."""
from llnl.util.tty.color import cwrite

97
lib/spack/spack/repository.py
View File

@@ -33,7 +33,7 @@
from external import yaml
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
from llnl.util.filesystem import *
import spack.error
import spack.config
@@ -316,6 +316,16 @@ def get(self, spec, new=False):
return self.repo_for_pkg(spec).get(spec)
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
return self.repo_for_pkg(spec).dump_provenance(spec, path)
def dirname_for_package_name(self, pkg_name):
return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
@@ -552,6 +562,35 @@ def get(self, spec, new=False):
return self._instances[key]
@_autospec
def dump_provenance(self, spec, path):
"""Dump provenance information for a spec to a particular path.
This dumps the package file and any associated patch files.
Raises UnknownPackageError if not found.
"""
# Some preliminary checks.
if spec.virtual:
raise UnknownPackageError(spec.name)
if spec.namespace and spec.namespace != self.namespace:
raise UnknownPackageError("Repository %s does not contain package %s."
% (self.namespace, spec.fullname))
# Install any patch files needed by packages.
mkdirp(path)
for spec, patches in spec.package.patches.items():
for patch in patches:
if patch.path:
if os.path.exists(patch.path):
install(patch.path, path)
else:
tty.warn("Patch file did not exist: %s" % patch.path)
# Install the package.py file itself.
install(self.filename_for_package_name(spec), path)
def purge(self):
"""Clear entire package instance cache."""
self._instances.clear()
@@ -705,6 +744,58 @@ def __contains__(self, pkg_name):
return self.exists(pkg_name)
def create_repo(root, namespace=None):
"""Create a new repository in root with the specified namespace.
If the namespace is not provided, use basename of root.
Return the canonicalized path and the namespace of the created repository.
"""
root = canonicalize_path(root)
if not namespace:
namespace = os.path.basename(root)
if not re.match(r'\w[\.\w-]*', namespace):
raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
existed = False
if os.path.exists(root):
if os.path.isfile(root):
raise BadRepoError('File %s already exists and is not a directory' % root)
elif os.path.isdir(root):
if not os.access(root, os.R_OK | os.W_OK):
raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
if os.listdir(root):
raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
existed = True
full_path = os.path.realpath(root)
parent = os.path.dirname(full_path)
if not os.access(parent, os.R_OK | os.W_OK):
raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
try:
config_path = os.path.join(root, repo_config_name)
packages_path = os.path.join(root, packages_dir_name)
mkdirp(packages_path)
with open(config_path, 'w') as config:
config.write("repo:\n")
config.write(" namespace: '%s'\n" % namespace)
except (IOError, OSError) as e:
raise BadRepoError('Failed to create new repository in %s.' % root,
"Caused by %s: %s" % (type(e), e))
# try to clean up.
if existed:
shutil.rmtree(config_path, ignore_errors=True)
shutil.rmtree(packages_path, ignore_errors=True)
else:
shutil.rmtree(root, ignore_errors=True)
return full_path, namespace
class RepoError(spack.error.SpackError):
"""Superclass for repository-related errors."""
@@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError):
"""Raised when there are no repositories configured."""
class InvalidNamespaceError(RepoError):
"""Raised when an invalid namespace is encountered."""
class BadRepoError(RepoError):
"""Raised when repo layout is invalid."""

7
lib/spack/spack/util/environment.py
View File

@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
for key in keys:
if key in dictionary:
dictionary.pop(key)
def dump_environment(path):
"""Dump the current environment out to a file."""
with open(path, 'w') as env_file:
for key,val in sorted(os.environ.items()):
env_file.write("%s=%s\n" % (key, val))